Files
notarius/internal/cli/assembled_spell_pipeline_contract_test.go

366 lines
16 KiB
Go

package cli
import (
"context"
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
spells "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
spellnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/spells"
"go.maximumdirect.net/promptkit"
)
const assembledSpellExtractorKey = "test/dnd/spell-casts"
const assembledDirectSpellValidatorKey = "test/dnd/direct-spell-correction"
func TestAssembledSpellPipelineNormalizesMergedCasts(t *testing.T) {
registries, resolved, extractor := assembledSpellPipeline(t, assembledSpellPipelineOptions{})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
chunkIndexes := extractor.chunkIndexesSnapshot()
sort.Ints(chunkIndexes)
if !reflect.DeepEqual(chunkIndexes, []int{0, 1}) {
t.Fatalf("extractor chunk indexes = %#v, want two chunk-boundary calls", chunkIndexes)
}
if len(output.OutputFiles) != 1 || output.FallbackCount != 0 {
t.Fatalf("output=%#v, want accepted spell artifact", output)
}
normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
if len(normalized.SpellCasts) != 2 {
t.Fatalf("normalized casts = %#v, want collapsed duplicate plus distinct evidence", normalized.SpellCasts)
}
first, distinct := normalized.SpellCasts[0], normalized.SpellCasts[1]
if first.Spell != "Cure Wounds" || first.Caster != " Aria \t" {
t.Fatalf("retained cast = %#v, want canonical spell with first occurrence caster", first)
}
if !reflect.DeepEqual(first.SourceRefs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, {SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
t.Fatalf("retained refs = %#v, want sorted complete evidence", first.SourceRefs)
}
if distinct.Spell != "Cure Wounds" || distinct.Caster != "aria" || !reflect.DeepEqual(distinct.SourceRefs, []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}) {
t.Fatalf("distinct cast = %#v, want separate evidence event", distinct)
}
wantDiagnosticReasons := []string{
spellnormalize.ReasonCodeSpellNameCanonicalized,
spellnormalize.ReasonCodeSourceReferencesNormalized,
spellnormalize.ReasonCodeDuplicateSpellCastCollapsed,
"spell_not_near_source",
}
gotDiagnosticReasons := make([]string, len(output.Diagnostics.Groups))
for index, group := range output.Diagnostics.Groups {
gotDiagnosticReasons[index] = group.ReasonCode
}
if !reflect.DeepEqual(gotDiagnosticReasons, wantDiagnosticReasons) {
t.Fatalf("diagnostics = %#v, want deterministic normalize and validation diagnostics", output.Diagnostics)
}
if output.Diagnostics.Groups[2].Samples[0].Scope != "spell_casts[0]" || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "retained input index 0") || !strings.Contains(output.Diagnostics.Groups[2].Samples[0].Message, "removed input indices [1]") {
t.Fatalf("duplicate diagnostic = %#v, want retained and removed merged indices", output.Diagnostics.Groups[2])
}
for _, group := range output.Diagnostics.Groups {
if group.Disposition == contracts.DiagnosticDispositionWarning {
t.Fatalf("advisory-only normalization emitted a process warning: %#v", group)
}
}
}
func TestAssembledSpellPipelineCorrectsRejectedDirectExtraction(t *testing.T) {
components := productionTestComponents(t)
if err := pipeline.RegisterTypedValidator[dnd.SpellList](components.registries.Validators, dnd.SpellListKind, pipeline.ValidatorSpec{Key: assembledDirectSpellValidatorKey, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[dnd.SpellList], error) { return assembledDirectSpellValidator{}, nil }); err != nil {
t.Fatal(err)
}
turns := 1
extract := pipeline.Binding(spells.Key)
extract.MaxAdditionalTurns = &turns
extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{pipeline.Binding(assembledDirectSpellValidatorKey)}}
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
ID: "assembled-dnd-correcting-spells", Input: pipeline.Binding("seriatim"), Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 2}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{"spells": {Extract: extract, Normalize: pipeline.Binding(spellnormalize.Key)}}, Output: pipeline.Binding("json"),
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
if err != nil {
t.Fatal(err)
}
client := &assembledSpellCorrectionClient{}
prepared, err := pipeline.Prepare(resolved, components.registries, pipeline.ModuleDependencies{Engine: productionNativeEngine(t, client)})
if err != nil {
t.Fatal(err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")})
if err != nil {
t.Fatal(err)
}
value := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
if output.FallbackCount != 0 || len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != "Cure Wounds" {
t.Fatalf("output=%#v spells=%#v", output, value)
}
client.mu.Lock()
requests := append([]promptkit.GenerateRequest(nil), client.requests...)
client.mu.Unlock()
if len(requests) != 2 {
t.Fatalf("provider calls=%d, want rejected and revised candidates", len(requests))
}
initial, revised := requests[0].Prompt.Messages, requests[1].Prompt.Messages
if len(revised) != len(initial)+2 || !reflect.DeepEqual(revised[:len(initial)], initial) {
t.Fatal("correction lost native growing history")
}
assistant, guidance := revised[len(initial)], revised[len(initial)+1]
if assistant.Role != promptkit.RoleAssistant || assistant.Content != assembledRejectedSpellResponse || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "use a known spell name") || !strings.Contains(guidance.Content, "complete replacement") {
t.Fatalf("native correction=%#v", revised[len(initial):])
}
for _, fragment := range []string{"unknown_spell", "spell is not in the catalog", assembledDirectSpellValidatorKey} {
if strings.Contains(guidance.Content, fragment) {
t.Fatalf("correction contains diagnostic prose or identifier: %s", guidance.Content)
}
}
}
func TestAssembledSpellPipelineHonorsNormalizeValidatorOverride(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true})
var normalizeChain *pipeline.ResolvedValidatorChain
for index := range resolved.ValidatorChains {
chain := &resolved.ValidatorChains[index]
if chain.Stage == pipeline.StageNormalize && chain.ModuleKey == spellnormalize.Key && chain.LaneID == "spells" {
normalizeChain = chain
break
}
}
if normalizeChain == nil || len(normalizeChain.Validators) != 1 || normalizeChain.Validators[0].Binding.Module != "generic/always_accept" {
t.Fatalf("normalize validator chain = %#v, want explicit always-accept override", normalizeChain)
}
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil || output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
t.Fatalf("Run() error = %v output = %#v, want approved override run", err, output)
}
for _, group := range output.Diagnostics.Groups {
if group.ReasonCode == "spell_not_near_source" {
t.Fatalf("diagnostics = %#v, want explicit validator override to replace default relatedness chain", output.Diagnostics)
}
}
}
func TestAssembledSpellPipelineRejectsUnknownSpellDuringNormalization(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{unknownSpell: true})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatal(err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json")})
for _, fragment := range []string{"normalize", `lane "spells"`, "extract/dnd/spells/catalog"} {
if err == nil || !strings.Contains(err.Error(), fragment) {
t.Fatalf("error=%v, want scoped catalog rejection (%s)", err, fragment)
}
}
if len(output.OutputFiles) != 0 || output.FallbackCount != 0 {
t.Fatalf("strict normalization failure published output: %#v", output)
}
}
func TestAssembledSpellPipelinePromotesUnknownSpellAdvisoryWhenOverrideAccepts(t *testing.T) {
registries, resolved, _ := assembledSpellPipeline(t, assembledSpellPipelineOptions{normalizeValidatorOverride: true, unknownSpell: true})
prepared, err := pipeline.Prepare(resolved, registries, pipeline.ModuleDependencies{})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
Prepared: prepared,
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if output.FallbackCount != 0 || len(output.OutputFiles) != 1 {
t.Fatalf("output=%#v, want accepted explicit validator override", output)
}
normalized := decodeAssembledOutput[dnd.SpellList](t, output.OutputFiles, "lanes/spells.json")
if len(normalized.SpellCasts) != 1 || normalized.SpellCasts[0].Spell != "Mysterious Burst" {
t.Fatalf("normalized casts = %#v, want unresolved name preserved", normalized.SpellCasts)
}
if len(output.Diagnostics.Groups) != 1 || output.Diagnostics.Groups[0].ReasonCode != spellnormalize.ReasonCodeSpellNameUnresolved || output.Diagnostics.Groups[0].Samples[0].Scope != "spell_casts[0]" {
t.Fatalf("diagnostics = %#v, want promoted scoped unresolved-name diagnostic", output.Diagnostics)
}
if output.Diagnostics.Groups[0].Disposition == contracts.DiagnosticDispositionWarning {
t.Fatal("unknown-spell advisory became a process warning")
}
}
type assembledSpellPipelineOptions struct {
normalizeValidatorOverride bool
unknownSpell bool
}
func assembledSpellPipeline(t *testing.T, options assembledSpellPipelineOptions) (pipeline.Registries, pipeline.ResolvedPipeline, *assembledSpellExtractor) {
t.Helper()
components := productionTestComponents(t)
extractor := &assembledSpellExtractor{unknownSpell: options.unknownSpell}
if err := pipeline.RegisterExtractor[dnd.SpellList](components.registries.Extractors, pipeline.ModuleSpec{
Key: assembledSpellExtractorKey,
Stage: pipeline.StageExtract,
ExecutionClass: contracts.ExecutionClassDeterministic,
Requires: []string{"chunks", "source.transcript"},
Provides: []string{"dnd.spell_casts"},
ArtifactKind: dnd.SpellListKind,
}, func() (contracts.Extractor[dnd.SpellList], error) {
return extractor, nil
}); err != nil {
t.Fatalf("register assembled extractor: %v", err)
}
normalize := pipeline.Binding(spellnormalize.Key)
if options.normalizeValidatorOverride {
normalize.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{pipeline.Binding("generic/always_accept")},
}
}
resolved, err := pipeline.ResolvePipeline(pipeline.PipelineProfile{
ID: "assembled-dnd-spells",
Input: pipeline.Binding("seriatim"),
Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"spells": {Extract: pipeline.Binding(assembledSpellExtractorKey), Normalize: normalize},
},
Output: pipeline.Binding("json"),
}, pipeline.ResolveOptions{}, catalogFromRegistries(components.registries))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
return components.registries, resolved, extractor
}
type assembledSpellExtractor struct {
mu sync.Mutex
chunkIndexes []int
unknownSpell bool
}
const assembledRejectedSpellResponse = `{"spell_casts":[{"caster":"Aria","spell":"Mysterious Burst","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
type assembledSpellCorrectionClient struct {
mu sync.Mutex
requests []promptkit.GenerateRequest
}
func (client *assembledSpellCorrectionClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
client.mu.Lock()
defer client.mu.Unlock()
if !productionSchemaHasProperty(request, "spell_casts") {
return nil, fmt.Errorf("unexpected native response contract")
}
client.requests = append(client.requests, request)
content := assembledRejectedSpellResponse
if len(client.requests) > 1 {
content = `{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`
}
return &promptkit.GenerateResponse{Content: content}, nil
}
type assembledDirectSpellValidator struct{}
func (assembledDirectSpellValidator) Name() string { return assembledDirectSpellValidatorKey }
func (assembledDirectSpellValidator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassDeterministic
}
func (assembledDirectSpellValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[dnd.SpellList]) (contracts.ValidationResult, error) {
for _, cast := range req.Value.SpellCasts {
if cast.Spell == "Mysterious Burst" {
return contracts.ValidationResult{Approved: false, ReasonCode: "unknown_spell", Message: "spell is not in the catalog", CorrectionGuidance: "use a known spell name"}, nil
}
}
return contracts.ValidationResult{Approved: true}, nil
}
func (e *assembledSpellExtractor) Key() string { return assembledSpellExtractorKey }
func (*assembledSpellExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (e *assembledSpellExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SpellList]) (contracts.TypedExtractionResult[dnd.SpellList], error) {
if err := ctx.Err(); err != nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, err
}
if req.Chunk == nil || req.Source == nil {
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("assembled extractor requires source and chunk")
}
e.mu.Lock()
e.chunkIndexes = append(e.chunkIndexes, req.Chunk.Index)
e.mu.Unlock()
refOne := source.SourceRef{SourceID: req.Source.ID, StartUnitID: 1, EndUnitID: 1}
refTwo := source.SourceRef{SourceID: req.Source.ID, StartUnitID: 2, EndUnitID: 2}
if e.unknownSpell {
if req.Chunk.Index == 0 {
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{{
Caster: "Aria", Spell: "Mysterious Burst", SourceRefs: []source.SourceRef{refOne},
}}}}, nil
}
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{}}}, nil
}
switch req.Chunk.Index {
case 0:
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{{
Caster: " Aria \t", Spell: " cure wounds ", SourceRefs: []source.SourceRef{refTwo, refOne},
}}}}, nil
case 1:
return contracts.TypedExtractionResult[dnd.SpellList]{Value: dnd.SpellList{SpellCasts: []dnd.SpellCast{
{Caster: "aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{refOne, refTwo}},
{Caster: "aria", Spell: "Cure Wounds", SourceRefs: []source.SourceRef{refTwo}},
}}}, nil
default:
return contracts.TypedExtractionResult[dnd.SpellList]{}, fmt.Errorf("unexpected assembled chunk index %d", req.Chunk.Index)
}
}
func (e *assembledSpellExtractor) chunkIndexesSnapshot() []int {
e.mu.Lock()
defer e.mu.Unlock()
return append([]int(nil), e.chunkIndexes...)
}
func decodeAssembledOutput[T any](t *testing.T, files []contracts.OutputFile, name string) T {
t.Helper()
for _, file := range files {
if file.Name != name {
continue
}
var value T
if err := json.Unmarshal(file.Bytes, &value); err != nil {
t.Fatalf("decode %s: %v", name, err)
}
return value
}
t.Fatalf("output files = %#v, want %q", files, name)
return *new(T)
}