Migrate assembled spell and catalog contracts to native correction
This commit is contained in:
@@ -1706,6 +1706,18 @@ legacy ConfigKit YAML import. Remaining CLI test migration, dependency tidy/
|
||||
old-path removal, notice reconciliation, and whole-tree validation are required
|
||||
before completion. This stage is not marked finished.
|
||||
|
||||
Assembled spell tests now exercise the real native extractor and continuation
|
||||
history rather than a synthetic model-candidate/correction protocol. Retained
|
||||
merged-cast ordering, canonicalization, evidence, explicit validator overrides,
|
||||
and process/advisory distinctions. Native catalog correction tests cover selected
|
||||
turn exhaustion and corrected overlay acceptance; strict terminal rejection
|
||||
publishes no artifact. Catalog materialization retains raw-reference provenance
|
||||
and semantic recognition across reordered overlay bytes, and the native CLI
|
||||
publishes the grounded overlay spell. Deleted only superseded checkpoint-identity/
|
||||
resume and manifest/sidecar assertions and their helpers. Focused assembled spell/
|
||||
catalog tests, vet, and repeated race checks pass; remaining CLI fixtures and final
|
||||
compilation are still required.
|
||||
|
||||
**Goal:** delete the old architecture rather than leave dormant alternatives.
|
||||
|
||||
**Inspect/change:** all production/test callers of the feature roadmap's removal
|
||||
|
||||
@@ -10,19 +10,17 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const assembledSpellExtractorKey = "test/dnd/spell-casts"
|
||||
|
||||
const assembledCorrectingSpellExtractorKey = "test/dnd/correcting-spell-casts"
|
||||
|
||||
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
|
||||
|
||||
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
@@ -33,9 +31,8 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
@@ -46,17 +43,10 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
if !reflect.DeepEqual(chunkIndexes, []int{0, 1}) {
|
||||
t.Fatalf("extractor chunk indexes = %#v, want two chunk-boundary calls", chunkIndexes)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want approved normalized output without rejections", output)
|
||||
}
|
||||
if output.NormalizeOutputs[0].NormalizerKey != spellnormalize.Key {
|
||||
t.Fatalf("normalized output module = %q, want %q", output.NormalizeOutputs[0].NormalizerKey, spellnormalize.Key)
|
||||
}
|
||||
|
||||
var normalized dnd.SpellList
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil {
|
||||
t.Fatalf("decode normalized output: %v", err)
|
||||
if len(output.OutputFiles) != 1 || output.FallbackCount != 0 {
|
||||
t.Fatalf("output=%#v, want accepted spell artifact", output)
|
||||
}
|
||||
normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
|
||||
if len(normalized.SpellCasts) != 2 {
|
||||
t.Fatalf("normalized casts = %#v, want collapsed duplicate plus distinct evidence", normalized.SpellCasts)
|
||||
}
|
||||
@@ -88,55 +78,60 @@ func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
|
||||
t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
|
||||
}
|
||||
|
||||
warningsFile := decodeAssembledOutput[struct {
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "warnings.json")
|
||||
if len(warningsFile.Groups) != 0 {
|
||||
t.Fatalf("warnings file = %#v, want no process warnings for advisory-only diagnostics", warningsFile.Groups)
|
||||
}
|
||||
diagnosticsFile := decodeAssembledOutput[struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
GroupCount int `json:"group_count"`
|
||||
OccurrenceCount int `json:"occurrence_count"`
|
||||
Truncated bool `json:"truncated"`
|
||||
UnrepresentedOccurrenceCount int `json:"unrepresented_occurrence_count"`
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "diagnostics.json")
|
||||
if diagnosticsFile.SchemaVersion != "notarius.diagnostics.v1" || diagnosticsFile.GroupCount != len(output.Diagnostics.Groups) || !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) || diagnosticsFile.OccurrenceCount != diagnosticGroupOccurrences(output.Diagnostics.Groups)+output.Diagnostics.UnrepresentedOccurrenceCount || diagnosticsFile.Truncated != output.Diagnostics.Truncated || diagnosticsFile.UnrepresentedOccurrenceCount != output.Diagnostics.UnrepresentedOccurrenceCount {
|
||||
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile, output.Diagnostics)
|
||||
}
|
||||
manifest := decodeAssembledOutput[artifacts.RunManifest](t, output.OutputFiles, "manifest.json")
|
||||
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].Normalizer != spellnormalize.Key {
|
||||
t.Fatalf("manifest lanes = %#v, want assembled spell normalizer", manifest.ArtifactLanes)
|
||||
}
|
||||
normalizerMetadata, ok := manifest.ArtifactLanes[0].Metadata["normalizer"].(map[string]any)
|
||||
_, hasOverlayIDs := normalizerMetadata["catalog_overlay_ids"]
|
||||
if !ok || normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !hasOverlayIDs {
|
||||
t.Fatalf("normalizer manifest metadata = %#v, want base ID, digest, and overlay IDs", manifest.ArtifactLanes[0].Metadata)
|
||||
for _, group := range output.Diagnostics.Groups {
|
||||
if group.Disposition == contracts.DiagnosticDispositionWarning {
|
||||
t.Fatalf("advisory-only normalization emitted a process warning: %#v", group)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
|
||||
registries, resolved, extractor := assembledCorrectingSpellPipeline(t)
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
components := productionTestComponents(t)
|
||||
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) { return assembledDirectSpellValidator{}, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
turns := 1
|
||||
extract := pipeline.Binding(spells.Key)
|
||||
extract.MaxAdditionalTurns = &turns
|
||||
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding(assembledDirectSpellValidatorKey)}}
|
||||
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||
ID: "assembled-dnd-correcting-spells", Input: pipeline.Binding("seriatim"), Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 2}},
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)}}, Output: pipeline.Binding("json"),
|
||||
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
client := &assembledSpellCorrectionClient{}
|
||||
prepared, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(output.Rejected) != 0 || output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want corrected accepted spell output", output)
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
correction := extractor.correctionSnapshot()
|
||||
if correction == nil || string(correction.AssistantResponse) != `{"spell":"Mysterious Burst"}` || !strings.Contains(correction.UserGuidance, "use a known spell name") || !strings.Contains(correction.UserGuidance, "complete corrected replacement") || strings.Contains(correction.UserGuidance, "unknown_spell") || strings.Contains(correction.UserGuidance, "spell is not in the catalog") {
|
||||
t.Fatalf("extract correction = %#v, want exact rejected model response and semantic replacement guidance only", correction)
|
||||
value := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
|
||||
if output.FallbackCount != 0 || len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" {
|
||||
t.Fatalf("output=%#v spells=%#v", output, value)
|
||||
}
|
||||
client.mu.Lock()
|
||||
requests := append([]promptkit.GenerateRequest(nil), client.requests...)
|
||||
client.mu.Unlock()
|
||||
if len(requests) != 2 {
|
||||
t.Fatalf("provider calls=%d, want rejected and revised candidates", len(requests))
|
||||
}
|
||||
initial, revised := requests[0].Prompt.Messages, requests[1].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) {
|
||||
t.Fatal("correction lost native growing history")
|
||||
}
|
||||
assistant, guidance := revised[len(initial)], revised[len(initial)+1]
|
||||
if assistant.Role != promptkit.RoleAssistant || assistant.Content != assembledRejectedSpellResponse || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "use a known spell name") || !strings.Contains(guidance.Content, "complete replacement") {
|
||||
t.Fatalf("native correction=%#v", revised[len(initial):])
|
||||
}
|
||||
for _, fragment := range []string{"unknown_spell", "spell is not in the catalog", assembledDirectSpellValidatorKey} {
|
||||
if strings.Contains(guidance.Content, fragment) {
|
||||
t.Fatalf("correction contains diagnostic prose or identifier: %s", guidance.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,11 +154,10 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
})
|
||||
if err != nil || output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
if err != nil || output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
|
||||
}
|
||||
for _, group := range output.Diagnostics.Groups {
|
||||
@@ -173,36 +167,20 @@ func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledSpellPipelinePromotesTerminalUnknownSpellDiagnostics(t *testing.T) {
|
||||
func TestAssembledSpellPipelineRejectsUnknownSpellDuringNormalization(t *testing.T) {
|
||||
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
|
||||
resolved.Steps[0].ArtifactLanes[0].NormalizeValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")})
|
||||
for _, fragment := range []string{"normalize", `lane "spells"`, "extract/dnd/spells/catalog"} {
|
||||
if err == nil || !strings.Contains(err.Error(), fragment) {
|
||||
t.Fatalf("error=%v, want scoped catalog rejection (%s)", err, fragment)
|
||||
}
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 || len(output.Rejected) != 1 {
|
||||
t.Fatalf("run output = %#v, want one rejected normalize candidate and no normalized output", output)
|
||||
}
|
||||
rejection := output.Rejected[0]
|
||||
if rejection.Stage != string(pipeline.StageNormalize) || rejection.LaneID != "spells" || rejection.ModuleKey != spellnormalize.Key || rejection.ValidatorName != "extract/dnd/spells/catalog" || rejection.ReasonCode != "unknown_spell" {
|
||||
t.Fatalf("rejection = %#v, want durable normalize catalog rejection", rejection)
|
||||
}
|
||||
rejectedFile := decodeAssembledOutput[struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected"`
|
||||
}](t, output.OutputFiles, "rejected.json")
|
||||
if !reflect.DeepEqual(rejectedFile.Rejected, output.Rejected) {
|
||||
t.Fatalf("rejected file = %#v, run rejections = %#v, want durable rejection diagnostic", rejectedFile.Rejected, output.Rejected)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" || output.Diagnostics.Groups[1].ReasonCode != "spell_not_near_source" {
|
||||
t.Fatalf("diagnostics = %#v, want complete terminal normalize validation diagnostics", output.Diagnostics)
|
||||
if len(output.OutputFiles) != 0 || output.FallbackCount != 0 {
|
||||
t.Fatalf("strict normalization failure published output: %#v", output)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -213,37 +191,24 @@ func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("run output = %#v, want accepted unknown spell with explicit validator override", output)
|
||||
}
|
||||
var normalized dnd.SpellList
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &normalized); err != nil {
|
||||
t.Fatalf("decode normalized output: %v", err)
|
||||
if output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("output=%#v, want accepted explicit validator override", output)
|
||||
}
|
||||
normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
|
||||
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
|
||||
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
|
||||
t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
|
||||
}
|
||||
warningsFile := decodeAssembledOutput[struct {
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "warnings.json")
|
||||
if len(warningsFile.Groups) != 0 {
|
||||
t.Fatalf("warnings file = %#v, want no process warnings for an advisory diagnostic", warningsFile.Groups)
|
||||
}
|
||||
diagnosticsFile := decodeAssembledOutput[struct {
|
||||
Groups []contracts.DiagnosticGroup `json:"groups"`
|
||||
}](t, output.OutputFiles, "diagnostics.json")
|
||||
if !reflect.DeepEqual(diagnosticsFile.Groups, output.Diagnostics.Groups) {
|
||||
t.Fatalf("diagnostics file = %#v, run diagnostics = %#v", diagnosticsFile.Groups, output.Diagnostics)
|
||||
if output.Diagnostics.Groups[0].Disposition == contracts.DiagnosticDispositionWarning {
|
||||
t.Fatal("unknown-spell advisory became a process warning")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,47 +217,6 @@ type assembledSpellPipelineOptions struct {
|
||||
unknownSpell bool
|
||||
}
|
||||
|
||||
func assembledCorrectingSpellPipeline(t *testing.T) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledCorrectingSpellExtractor) {
|
||||
t.Helper()
|
||||
components := productionTestComponents(t)
|
||||
extractor := &assembledCorrectingSpellExtractor{}
|
||||
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
|
||||
Key: assembledCorrectingSpellExtractorKey,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"dnd.spell_casts"},
|
||||
ArtifactKind: dnd.SpellListKind,
|
||||
}, func() (contracts.Extractor[dnd.SpellList], error) {
|
||||
return extractor, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register correcting extractor: %v", err)
|
||||
}
|
||||
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) {
|
||||
return assembledDirectSpellValidator{}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("register direct spell validator: %v", err)
|
||||
}
|
||||
|
||||
extract := pipeline.Binding(assembledCorrectingSpellExtractorKey)
|
||||
extract.Retries = 1
|
||||
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: assembledDirectSpellValidatorKey}}}
|
||||
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
|
||||
ID: "assembled-dnd-correcting-spells",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)},
|
||||
},
|
||||
Output: pipeline.Binding("json"),
|
||||
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
return components.registries, resolved, extractor
|
||||
}
|
||||
|
||||
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
|
||||
t.Helper()
|
||||
components := productionTestComponents(t)
|
||||
@@ -338,50 +262,28 @@ type assembledSpellExtractor struct {
|
||||
unknownSpell bool
|
||||
}
|
||||
|
||||
type assembledCorrectingSpellExtractor struct {
|
||||
mu sync.Mutex
|
||||
correction *contracts.SemanticCorrection
|
||||
const assembledRejectedSpellResponse = `{"spell_casts":[{"caster":"Aria","spell":"Mysterious Burst","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||||
|
||||
type assembledSpellCorrectionClient struct {
|
||||
mu sync.Mutex
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (*assembledCorrectingSpellExtractor) Key() string { return assembledCorrectingSpellExtractorKey }
|
||||
|
||||
func (*assembledCorrectingSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (e *assembledCorrectingSpellExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
if req.Source == nil || req.Chunk == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("correcting assembled extractor requires source and chunk")
|
||||
func (client *assembledSpellCorrectionClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response := `{"spell":"accepted"}`
|
||||
value := dnd.SpellList{SpellCasts: []dnd.SpellCast{}}
|
||||
if req.Chunk.Index == 0 && req.Correction == nil {
|
||||
response = `{"spell":"Mysterious Burst"}`
|
||||
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
if !productionSchemaHasProperty(request, "spell_casts") {
|
||||
return nil, fmt.Errorf("unexpected native response contract")
|
||||
}
|
||||
if req.Chunk.Index == 0 && req.Correction != nil {
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.correction = correction
|
||||
e.mu.Unlock()
|
||||
value.SpellCasts = []dnd.SpellCast{{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}}}}
|
||||
client.requests = append(client.requests, request)
|
||||
content := assembledRejectedSpellResponse
|
||||
if len(client.requests) > 1 {
|
||||
content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
|
||||
}
|
||||
candidate, err := contracts.NewModelCandidate([]byte(response), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{Value: value, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
func (e *assembledCorrectingSpellExtractor) correctionSnapshot() *contracts.SemanticCorrection {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
correction, err := contracts.CloneSemanticCorrection(e.correction)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return correction
|
||||
return &promptkit.GenerateResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
type assembledDirectSpellValidator struct{}
|
||||
@@ -405,7 +307,7 @@ func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorK
|
||||
|
||||
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
|
||||
}
|
||||
@@ -446,14 +348,6 @@ func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
|
||||
return append([]int(nil), e.chunkIndexes...)
|
||||
}
|
||||
|
||||
func diagnosticGroupOccurrences(groups []contracts.DiagnosticGroup) int {
|
||||
count := 0
|
||||
for _, group := range groups {
|
||||
count += group.OccurrenceCount
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
|
||||
t.Helper()
|
||||
for _, file := range files {
|
||||
|
||||
@@ -5,19 +5,19 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
const retries = 2
|
||||
func TestProductionSpellCatalogValidationCoordinatesCorrections(t *testing.T) {
|
||||
const additionalTurns = 2
|
||||
tests := []struct {
|
||||
name string
|
||||
responses []string
|
||||
@@ -33,11 +33,11 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
},
|
||||
wantCalls: retries + 1,
|
||||
wantCalls: additionalTurns + 1,
|
||||
wantRejected: true,
|
||||
},
|
||||
{
|
||||
name: "overlay spell becomes valid on retry",
|
||||
name: "overlay spell becomes valid after correction",
|
||||
responses: []string{
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Aegis of Emberfall"),
|
||||
@@ -53,6 +53,12 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
cfg := loadMaintainedExample(t, configPath)
|
||||
profile := cfg.Pipelines["dnd-session"]
|
||||
lane := profile.Artifacts["spells"]
|
||||
turns := additionalTurns
|
||||
lane.Extract.MaxAdditionalTurns = &turns
|
||||
profile.Artifacts["spells"] = lane
|
||||
cfg.Pipelines["dnd-session"] = profile
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production configuration: %v", err)
|
||||
@@ -64,56 +70,36 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("materialize production references: %v", err)
|
||||
}
|
||||
materialized.Steps[0].ArtifactLanes[0].Extract.Retries = retries
|
||||
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
|
||||
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
llmClient := &catalogCorrectionLLMClient{responses: tt.responses}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, llmClient)})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare production pipeline: %v", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if calls := llmClient.CallCount(); calls > retries+1 || calls != tt.wantCalls {
|
||||
t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, retries+1)
|
||||
if calls := llmClient.CallCount(); calls > additionalTurns+1 || calls != tt.wantCalls {
|
||||
t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, additionalTurns+1)
|
||||
}
|
||||
|
||||
if tt.wantRejected {
|
||||
if len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("rejected = %#v normalized = %#v, want one nonfatal rejection and no merge output", output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
rejection := output.Rejected[0]
|
||||
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
||||
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != "spell_not_near_source" {
|
||||
t.Fatalf("diagnostics = %#v, want complete terminal validation diagnostic", output.Diagnostics)
|
||||
if err == nil || !strings.Contains(err.Error(), "extract/dnd/spells/catalog") || len(output.OutputFiles) != 0 || output.FallbackCount != 0 {
|
||||
t.Fatalf("output=%#v error=%v, want strict terminal catalog rejection", output, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("rejected = %#v normalized = %#v, want only accepted output", output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
var value dnd.SpellList
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &value); err != nil {
|
||||
t.Fatalf("decode normalized spell list: %v", err)
|
||||
if err != nil || output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
|
||||
t.Fatalf("output=%#v error=%v", output, err)
|
||||
}
|
||||
value := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
|
||||
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
||||
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 2 || output.Diagnostics.Groups[0].ReasonCode != tt.wantAdvisoryCode {
|
||||
t.Fatalf("diagnostics = %#v, want terminal extract and normalize diagnostics", output.Diagnostics)
|
||||
}
|
||||
if len(output.Diagnostics.Groups) != 2 {
|
||||
t.Fatalf("diagnostics = %#v, want extract and normalize data-quality advisories", output.Diagnostics)
|
||||
}
|
||||
for _, diagnostic := range output.Diagnostics.Groups {
|
||||
if diagnostic.Disposition != contracts.DiagnosticDispositionAdvisory || diagnostic.ReasonCode != tt.wantAdvisoryCode {
|
||||
t.Fatalf("diagnostics = %#v, want only data-quality advisories", output.Diagnostics)
|
||||
@@ -123,47 +109,43 @@ func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type catalogRetryLLMClient struct {
|
||||
type catalogCorrectionLLMClient struct {
|
||||
mu sync.Mutex
|
||||
responses []string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (client *catalogRetryLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
func (client *catalogCorrectionLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
if req.PromptID != spells.PromptID {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
|
||||
if !productionSchemaHasProperty(request, "spell_casts") {
|
||||
return nil, fmt.Errorf("unexpected native spell response contract")
|
||||
}
|
||||
client.mu.Lock()
|
||||
index := client.calls
|
||||
client.calls++
|
||||
client.mu.Unlock()
|
||||
if index >= len(client.responses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
|
||||
return nil, fmt.Errorf("missing fake response %d", index)
|
||||
}
|
||||
content := []byte(client.responses[index])
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
||||
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
|
||||
}
|
||||
|
||||
func (client *catalogRetryLLMClient) CallCount() int {
|
||||
func (client *catalogCorrectionLLMClient) CallCount() int {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
return client.calls
|
||||
}
|
||||
|
||||
func productionSpellResponse(name string) string {
|
||||
content, err := json.Marshal(dnd.SpellList{SpellCasts: []dnd.SpellCast{{
|
||||
Caster: "Aria",
|
||||
Spell: name,
|
||||
SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
|
||||
content, err := json.Marshal(map[string]any{"spell_casts": []any{map[string]any{
|
||||
"caster": "Aria", "spell": name, "source_refs": []any{map[string]int{"start_unit_id": 1, "end_unit_id": 1}},
|
||||
}}})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(content)
|
||||
}
|
||||
|
||||
var _ promptkit.LLMClient = (*catalogCorrectionLLMClient)(nil)
|
||||
@@ -1,8 +1,6 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,100 +8,69 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
|
||||
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
|
||||
)
|
||||
|
||||
func TestSpellCatalogBytesAffectCheckpointIdentityButNotSemanticDigest(t *testing.T) {
|
||||
func TestMaterializedSpellCatalogOrderingPreservesSemanticContents(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
catalog := catalogFromRegistries(components.registries)
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production configuration: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
|
||||
resolved := effective.ResolvedPipeline
|
||||
bindings := resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings
|
||||
catalogBindingIndex := -1
|
||||
for index, binding := range bindings {
|
||||
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
catalogBindingIndex = index
|
||||
break
|
||||
overlayPath := filepath.Join(t.TempDir(), "catalog.json")
|
||||
for _, target := range []*pipeline.ResolvedReferenceTarget{&resolved.Steps[0].ArtifactLanes[0].ExtractReferences, &resolved.Steps[0].ArtifactLanes[0].NormalizeReferences} {
|
||||
matched := false
|
||||
for index := range target.Bindings {
|
||||
if target.Bindings[index].SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
target.Bindings[index].Source = overlayPath
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
t.Fatal("missing configured spell catalog binding")
|
||||
}
|
||||
}
|
||||
if catalogBindingIndex < 0 {
|
||||
t.Fatalf("spell catalog bindings = %#v, want catalog binding", bindings)
|
||||
}
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings[catalogBindingIndex].Source = overlayPath
|
||||
normalizeBindings := resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings
|
||||
normalizeCatalogBindingIndex := -1
|
||||
for index, binding := range normalizeBindings {
|
||||
if binding.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
normalizeCatalogBindingIndex = index
|
||||
break
|
||||
materialize := func(contents string) pipeline.ResolvedPipeline {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(overlayPath, []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, _, err := pipeline.MaterializeReferences(resolved, catalog, pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
if normalizeCatalogBindingIndex < 0 {
|
||||
t.Fatalf("normalize spell catalog bindings = %#v, want catalog binding", normalizeBindings)
|
||||
}
|
||||
resolved.Steps[0].ArtifactLanes[0].NormalizeReferences.Bindings[normalizeCatalogBindingIndex].Source = overlayPath
|
||||
|
||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayA), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materializedA, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize first catalog: %v", err)
|
||||
}
|
||||
identityA := catalogCheckpointIdentity(t, materializedA)
|
||||
metadataA := catalogExtractorMetadata(t, materializedA)
|
||||
normalizerMetadataA := catalogNormalizerMetadata(t, materializedA)
|
||||
referenceA := catalogReference(t, materializedA)
|
||||
|
||||
if err := os.WriteFile(overlayPath, []byte(reorderedOverlayB), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materializedB, _, err := pipeline.MaterializeReferences(resolved, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize reordered catalog: %v", err)
|
||||
}
|
||||
identityB := catalogCheckpointIdentity(t, materializedB)
|
||||
metadataB := catalogExtractorMetadata(t, materializedB)
|
||||
normalizerMetadataB := catalogNormalizerMetadata(t, materializedB)
|
||||
referenceB := catalogReference(t, materializedB)
|
||||
|
||||
if identityA.Digest == identityB.Digest {
|
||||
t.Fatalf("checkpoint identity digest = %q for both raw catalog files, want invalidation", identityA.Digest)
|
||||
}
|
||||
if referenceA.Digest == referenceB.Digest || referenceA.OriginURI != referenceB.OriginURI {
|
||||
t.Fatalf("catalog reference provenance changed from %#v to %#v, want same origin and different raw digest", referenceA, referenceB)
|
||||
}
|
||||
digestA, ok := metadataA["catalog_digest"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("first extractor catalog metadata = %#v, want digest", metadataA)
|
||||
}
|
||||
digestB, ok := metadataB["catalog_digest"].(string)
|
||||
if !ok || digestA != digestB {
|
||||
t.Fatalf("extractor catalog digests = %q and %q, want same semantic digest", digestA, digestB)
|
||||
}
|
||||
if got, want := metadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(metadataB["catalog_overlay_ids"], want) {
|
||||
t.Fatalf("extractor overlay IDs = %#v and %#v, want %#v", got, metadataB["catalog_overlay_ids"], want)
|
||||
}
|
||||
normalizerDigestA, ok := normalizerMetadataA["catalog_digest"].(string)
|
||||
normalizerDigestB, okB := normalizerMetadataB["catalog_digest"].(string)
|
||||
if !ok || !okB || normalizerDigestA != digestA || normalizerDigestB != digestB {
|
||||
t.Fatalf("normalizer catalog digests = %#v and %#v, want extractor semantic digests %q and %q", normalizerMetadataA["catalog_digest"], normalizerMetadataB["catalog_digest"], digestA, digestB)
|
||||
}
|
||||
if got, want := normalizerMetadataA["catalog_overlay_ids"], []string{"campaign.a", "campaign.b"}; !reflect.DeepEqual(got, want) || !reflect.DeepEqual(normalizerMetadataB["catalog_overlay_ids"], want) {
|
||||
t.Fatalf("normalizer overlay IDs = %#v and %#v, want %#v", got, normalizerMetadataB["catalog_overlay_ids"], want)
|
||||
first, second := materialize(reorderedOverlayA), materialize(reorderedOverlayB)
|
||||
laneA, laneB := first.Steps[0].ArtifactLanes[0], second.Steps[0].ArtifactLanes[0]
|
||||
for _, pair := range [][2]pipeline.ResolvedReferenceTarget{
|
||||
{laneA.ExtractReferences, laneB.ExtractReferences},
|
||||
{laneA.NormalizeReferences, laneB.NormalizeReferences},
|
||||
} {
|
||||
itemsA, itemsB := pair[0].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items, pair[1].ReferenceSet.Slots[spellcatalog.SpellCatalogReferenceSlot].Items
|
||||
if len(itemsA) != 1 || len(itemsB) != 1 {
|
||||
t.Fatal("materialized catalog missing")
|
||||
}
|
||||
if itemsA[0].Digest == itemsB[0].Digest || itemsA[0].Origin != itemsB[0].Origin {
|
||||
t.Fatalf("raw catalog provenance=%#v/%#v", itemsA, itemsB)
|
||||
}
|
||||
catalogA, err := spellcatalog.ResolveEffectiveCatalog(pair[0].ReferenceSet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalogB, err := spellcatalog.ResolveEffectiveCatalog(pair[1].ReferenceSet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if catalogA.Digest() != catalogB.Digest() || !reflect.DeepEqual(catalogA.OverlayIDs(), []string{"campaign.a", "campaign.b"}) || !reflect.DeepEqual(catalogB.OverlayIDs(), catalogA.OverlayIDs()) || !reflect.DeepEqual(catalogA.PromptSpells(), catalogB.PromptSpells()) {
|
||||
t.Fatal("catalog ordering changed semantic recognition")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,11 +90,11 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
|
||||
t.Fatal(err)
|
||||
}
|
||||
components := productionTestComponents(t)
|
||||
first, err := loadMaintainedExample(t, firstPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
first, err := loadMaintainedExample(t, firstPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve first configuration: %v", err)
|
||||
}
|
||||
second, err := loadMaintainedExample(t, secondPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
second, err := loadMaintainedExample(t, secondPath).Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve changed configuration: %v", err)
|
||||
}
|
||||
@@ -136,360 +103,39 @@ func TestConfiguredSpellCatalogBindingChangesResolvedPipelineIdentity(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticSpellCatalogFingerprintChangesCheckpointIdentityWithoutReferenceChange(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints := prepared.CheckpointFingerprints()
|
||||
wantNames := map[string]struct{}{
|
||||
"extract:spells:" + spells.Key + ":effective_catalog": {},
|
||||
"extract:spells:" + spells.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
|
||||
"normalize:spells:" + spellnormalize.Key + ":effective_catalog": {},
|
||||
"normalize:spells:" + spellnormalize.Key + ":validator:2:extract/dnd/spells/catalog:effective_catalog": {},
|
||||
}
|
||||
seen := make(map[string]string, len(fingerprints))
|
||||
for _, fingerprint := range fingerprints {
|
||||
if _, ok := wantNames[fingerprint.Name]; ok {
|
||||
seen[fingerprint.Name] = fingerprint.Value
|
||||
}
|
||||
}
|
||||
if len(seen) != len(wantNames) {
|
||||
t.Fatalf("prepared fingerprints = %#v, want scoped extractor and normalize catalog identities", fingerprints)
|
||||
}
|
||||
var catalogDigest string
|
||||
for name, value := range seen {
|
||||
if catalogDigest == "" {
|
||||
catalogDigest = value
|
||||
} else if value != catalogDigest {
|
||||
t.Fatalf("prepared fingerprint %q = %q, want shared semantic catalog digest %q", name, value, catalogDigest)
|
||||
}
|
||||
}
|
||||
|
||||
identityFor := func(values []pipeline.CheckpointFingerprint) checkpoint.Identity {
|
||||
identity, identityErr := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: materialized,
|
||||
InputKey: materialized.Input.Module,
|
||||
RawInputDigest: "sha256:unchanged-input",
|
||||
References: pipeline.ReferenceProvenance(materialized),
|
||||
ProvenanceFingerprints: checkpointIdentityFingerprints(values),
|
||||
})
|
||||
if identityErr != nil {
|
||||
t.Fatal(identityErr)
|
||||
}
|
||||
return identity
|
||||
}
|
||||
first := identityFor(fingerprints)
|
||||
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
|
||||
second := identityFor(changed)
|
||||
if first.Digest == second.Digest || reflect.DeepEqual(first.ReferenceDigests, nil) || !reflect.DeepEqual(first.ReferenceDigests, second.ReferenceDigests) {
|
||||
t.Fatalf("identities = %#v / %#v, want semantic invalidation with unchanged reference provenance", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := writeProductionSpellCatalogContractConfig(t)
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{ConfigPath: configPath, WorkingDir: filepath.Dir(configPath)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints := prepared.CheckpointFingerprints()
|
||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}}
|
||||
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
||||
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc := source.SourceDocument{ID: "source", Kind: "transcript", Format: "application/json"}
|
||||
doc.Units = []source.SourceUnit{{ID: 1, Kind: "turn", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 1}}}
|
||||
doc.Digest, err = source.DigestDocument(&doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recorder.SourceSucceeded(materialized.Input.Module, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalizeDependencies := []pipeline.CheckpointFingerprint{{Name: "artifact[0]", Value: "sha256:merged-artifact"}}
|
||||
normalizeSchema := contracts.ArtifactSchema{ID: "notarius.dnd.spells", Name: "notarius_dnd_spells", Version: "v1"}
|
||||
normalizeArtifact := pipeline.CheckpointArtifact{
|
||||
LaneID: "spells", ModuleKey: spellnormalize.Key, SourceID: doc.ID,
|
||||
SchemaDigest: contracts.DigestArtifactSchema(normalizeSchema),
|
||||
Artifact: contracts.SerializedArtifact{
|
||||
Kind: dnd.SpellListKind, Schema: normalizeSchema, MediaType: "application/json", Content: []byte(`{"spell_casts":[]}`),
|
||||
},
|
||||
}
|
||||
if err := recorder.NormalizeSucceeded("spells", spellnormalize.Key, normalizeDependencies, normalizeArtifact); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, sameLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := sameLoader.Source(materialized.Input.Module); !decision.Reused {
|
||||
t.Fatalf("same fingerprint decision = %#v, want reuse", decision)
|
||||
}
|
||||
if restored, decision := sameLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); !decision.Reused || string(restored.Output.Artifact.Content) != `{"spell_casts":[]}` {
|
||||
t.Fatalf("same normalize checkpoint = %#v, decision=%#v, want reuse", restored, decision)
|
||||
}
|
||||
changed := replaceCheckpointFingerprintValue(t, fingerprints, normalizeSpellCatalogFingerprintName(), "sha256:changed-effective-catalog")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changed, normalizeSpellCatalogFingerprintName())
|
||||
_, changedLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changed, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := changedLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed fingerprint decision = %#v, want cold miss", decision)
|
||||
}
|
||||
if _, decision := changedLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed normalize fingerprint decision = %#v, want normalize checkpoint cold miss", decision)
|
||||
}
|
||||
changedMapping := replaceCheckpointFingerprintValue(t, fingerprints, extractSpellMappingFingerprintName(), "dnd.spells.extract_mapping.changed")
|
||||
assertOnlyCheckpointFingerprintChanged(t, fingerprints, changedMapping, extractSpellMappingFingerprintName())
|
||||
_, mappingLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, changedMapping, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := mappingLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
||||
}
|
||||
|
||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}}
|
||||
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision)
|
||||
}
|
||||
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeSpellCatalogFingerprintName() string {
|
||||
return "normalize:spells:" + spellnormalize.Key + ":effective_catalog"
|
||||
}
|
||||
|
||||
func extractSpellMappingFingerprintName() string {
|
||||
return "extract:spells:" + spells.Key + ":mapping_policy"
|
||||
}
|
||||
|
||||
func replaceCheckpointFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, value string) []pipeline.CheckpointFingerprint {
|
||||
t.Helper()
|
||||
changed := append([]pipeline.CheckpointFingerprint(nil), fingerprints...)
|
||||
matches := 0
|
||||
for index := range changed {
|
||||
if changed[index].Name == name {
|
||||
changed[index].Value = value
|
||||
matches++
|
||||
}
|
||||
}
|
||||
if matches != 1 {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want exactly one fingerprint named %q", fingerprints, name)
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
func assertOnlyCheckpointFingerprintChanged(t *testing.T, before, after []pipeline.CheckpointFingerprint, changedName string) {
|
||||
t.Helper()
|
||||
if len(before) != len(after) {
|
||||
t.Fatalf("fingerprint lengths = %d and %d, want equal", len(before), len(after))
|
||||
}
|
||||
changes := 0
|
||||
for index := range before {
|
||||
if before[index].Name != after[index].Name {
|
||||
t.Fatalf("fingerprint[%d] name changed from %q to %q", index, before[index].Name, after[index].Name)
|
||||
}
|
||||
if before[index].Value == after[index].Value {
|
||||
continue
|
||||
}
|
||||
changes++
|
||||
if before[index].Name != changedName {
|
||||
t.Fatalf("fingerprint %q changed unexpectedly", before[index].Name)
|
||||
}
|
||||
}
|
||||
if changes != 1 {
|
||||
t.Fatalf("fingerprints changed %d values, want exactly %q", changes, changedName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedProductionOverlayRunAlignsGroundingValidationAndProvenance(t *testing.T) {
|
||||
func TestProductionOverlayRunAlignsGroundingAndValidation(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
fake := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
|
||||
options := productionRunOptions(t, fake)
|
||||
client := &productionFakeLLMClient{spellResponse: productionSpellResponse("Aegis of Emberfall")}
|
||||
configPath := writeProductionContractConfig(t, productionSpellCatalogContractConfig(t)+productionProviderConfiguration)
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", writeProductionSpellCatalogContractConfig(t),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
||||
}, &stdout, &stderr, options)
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input-file", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--lane", "spells", "--cache", "bypass", "--output-dir", outputRoot, "--format", "json"}, &stdout, &stderr, productionRunOptions(t, client))
|
||||
if code != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
runRoot := filepath.Join(outputRoot, productionRunID)
|
||||
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
||||
if manifest.ValidationStatus != "approved" || len(manifest.References) == 0 || len(manifest.ArtifactLanes) != 1 {
|
||||
t.Fatalf("manifest = %#v, want approved overlay run with one lane and references", manifest)
|
||||
}
|
||||
lane := manifest.ArtifactLanes[0]
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("lane metadata = %#v, want extractor metadata", lane.Metadata)
|
||||
}
|
||||
if extractorMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(extractorMetadata["catalog_digest"]), "sha256:") {
|
||||
t.Fatalf("extractor catalog metadata = %#v, want base ID and semantic digest", extractorMetadata)
|
||||
}
|
||||
if got := stringValues(extractorMetadata["catalog_overlay_ids"]); !reflect.DeepEqual(got, []string{"notarius.example-campaign"}) {
|
||||
t.Fatalf("catalog overlay IDs = %#v, want maintained overlay", got)
|
||||
}
|
||||
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("lane metadata = %#v, want normalizer metadata", lane.Metadata)
|
||||
}
|
||||
if normalizerMetadata["catalog_base_id"] != spellcatalog.SRD5E2014ID || !strings.HasPrefix(stringValue(normalizerMetadata["catalog_digest"]), "sha256:") || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), []string{"notarius.example-campaign"}) {
|
||||
t.Fatalf("normalizer catalog metadata = %#v, want base ID, semantic digest, and overlay IDs", normalizerMetadata)
|
||||
}
|
||||
if normalizerMetadata["catalog_digest"] != extractorMetadata["catalog_digest"] || !reflect.DeepEqual(stringValues(normalizerMetadata["catalog_overlay_ids"]), stringValues(extractorMetadata["catalog_overlay_ids"])) {
|
||||
t.Fatalf("extractor metadata = %#v, normalizer metadata = %#v, want shared catalog identity", extractorMetadata, normalizerMetadata)
|
||||
}
|
||||
|
||||
var catalogProvenances []artifacts.ReferenceProvenance
|
||||
for index := range manifest.References {
|
||||
reference := &manifest.References[index]
|
||||
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot {
|
||||
catalogProvenances = append(catalogProvenances, *reference)
|
||||
}
|
||||
}
|
||||
if len(catalogProvenances) != 2 {
|
||||
t.Fatalf("manifest references = %#v, want independently materialized extract and normalize catalog provenance", manifest.References)
|
||||
}
|
||||
overlayBytes := readRepositoryFile(t, "examples", "dnd-spell-catalog.json")
|
||||
for _, catalogProvenance := range catalogProvenances {
|
||||
if catalogProvenance.Stage != "extract" && catalogProvenance.Stage != "normalize" {
|
||||
t.Fatalf("catalog provenance = %#v, want extract or normalize scope", catalogProvenance)
|
||||
}
|
||||
if catalogProvenance.LaneID != "spells" || catalogProvenance.OriginType != "file" || catalogProvenance.MediaType != "application/json" || catalogProvenance.SizeBytes != int64(len(overlayBytes)) || catalogProvenance.Digest != digestBytes(overlayBytes) || !strings.Contains(catalogProvenance.OriginURI, "dnd-spell-catalog.json") {
|
||||
t.Fatalf("catalog provenance = %#v, want raw overlay provenance in both scopes", catalogProvenance)
|
||||
}
|
||||
}
|
||||
manifestBytes, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
var report runResult
|
||||
if err := json.Unmarshal([]byte(stdout.String()), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, leaked := range []string{"Aegis of Emberfall", "Emberfall Aegis", "Notarius example campaign spell names"} {
|
||||
if strings.Contains(string(manifestBytes), leaked) {
|
||||
t.Fatalf("manifest leaked overlay content %q", leaked)
|
||||
}
|
||||
if report.ArtifactCount != 1 || report.FallbackCount != 0 || report.OutputDirectory != filepath.Join(outputRoot, productionRunID) {
|
||||
t.Fatalf("completion=%#v", report)
|
||||
}
|
||||
|
||||
requests := fake.requestsFor(spells.PromptID)
|
||||
requests := client.requestsWithProperty("spell_casts")
|
||||
if len(requests) != 1 {
|
||||
t.Fatalf("spell requests = %d, want one", len(requests))
|
||||
t.Fatalf("native spell requests=%d", len(requests))
|
||||
}
|
||||
catalogInput, ok := requests[0].Inputs[spellcatalog.SpellCatalogReferenceSlot]
|
||||
if !ok || !strings.Contains(string(catalogInput.Content), `"canonical_name":"Aegis of Emberfall"`) || !strings.Contains(string(catalogInput.Content), `"aliases":["Emberfall Aegis"]`) {
|
||||
t.Fatalf("spell catalog prompt input = %#v, want canonical overlay name and recognition alias", catalogInput)
|
||||
var prompt strings.Builder
|
||||
for _, message := range requests[0].Prompt.Messages {
|
||||
prompt.WriteString(message.Content)
|
||||
}
|
||||
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, "lanes", "spells.json"))
|
||||
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
||||
t.Fatalf("artifact = %#v, want accepted overlay-only canonical spell", artifact)
|
||||
}
|
||||
rejected := readProductionJSON[struct {
|
||||
Rejected []json.RawMessage `json:"rejected"`
|
||||
}](t, filepath.Join(runRoot, "rejected.json"))
|
||||
if len(rejected.Rejected) != 0 {
|
||||
t.Fatalf("rejected = %#v, want no rejected output", rejected.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func catalogCheckpointIdentity(t *testing.T, resolved pipeline.ResolvedPipeline) checkpoint.Identity {
|
||||
t.Helper()
|
||||
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
||||
Pipeline: resolved,
|
||||
InputKey: resolved.Input.Module,
|
||||
RawInputDigest: "sha256:catalog-test-input",
|
||||
References: pipeline.ReferenceProvenance(resolved),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create checkpoint identity: %v", err)
|
||||
}
|
||||
return identity
|
||||
}
|
||||
|
||||
func catalogExtractorMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
||||
t.Helper()
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
extractor, err := spells.New(&productionFakeLLMClient{}, spells.Options{}, lane.ExtractReferences.ReferenceSet)
|
||||
if err != nil {
|
||||
t.Fatalf("construct extractor: %v", err)
|
||||
}
|
||||
return extractor.ManifestMetadata()
|
||||
}
|
||||
|
||||
func catalogNormalizerMetadata(t *testing.T, resolved pipeline.ResolvedPipeline) map[string]any {
|
||||
t.Helper()
|
||||
lane := resolved.Steps[0].ArtifactLanes[0]
|
||||
normalizer, err := spellnormalize.New(spellnormalize.Options{}, lane.NormalizeReferences.ReferenceSet)
|
||||
if err != nil {
|
||||
t.Fatalf("construct normalizer: %v", err)
|
||||
}
|
||||
return normalizer.ManifestMetadata()
|
||||
}
|
||||
|
||||
func catalogReference(t *testing.T, resolved pipeline.ResolvedPipeline) artifacts.ReferenceProvenance {
|
||||
t.Helper()
|
||||
for _, reference := range pipeline.ReferenceProvenance(resolved) {
|
||||
if reference.SlotName == spellcatalog.SpellCatalogReferenceSlot && reference.Stage == "extract" && reference.LaneID == "spells" {
|
||||
return reference
|
||||
for _, fragment := range []string{`"canonical_name":"Aegis of Emberfall"`, `"aliases":["Emberfall Aegis"]`} {
|
||||
if !strings.Contains(prompt.String(), fragment) {
|
||||
t.Fatalf("catalog prompt omitted overlay recognition %q", fragment)
|
||||
}
|
||||
}
|
||||
t.Fatalf("resolved references = %#v, want spell catalog provenance", pipeline.ReferenceProvenance(resolved))
|
||||
return artifacts.ReferenceProvenance{}
|
||||
}
|
||||
|
||||
func stringValue(value any) string {
|
||||
result, _ := value.(string)
|
||||
return result
|
||||
}
|
||||
|
||||
func stringValues(value any) []string {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil
|
||||
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(report.OutputDirectory, "lanes", "spells.json"))
|
||||
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Aegis of Emberfall" {
|
||||
t.Fatalf("spell artifact=%#v", artifact)
|
||||
}
|
||||
var values []string
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return nil
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func digestBytes(value []byte) string {
|
||||
sum := sha256.Sum256(value)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
const reorderedOverlayA = `{
|
||||
|
||||
Reference in New Issue
Block a user