Files
notarius/internal/modules/dnd/extract/spells/extractor_test.go

369 lines
17 KiB
Go

package spells
import (
"context"
"encoding/json"
"errors"
"reflect"
"sort"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
spellcatalog "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/spells/catalog"
"go.maximumdirect.net/promptkit"
)
func TestExtractReturnsCanonicalSpellListFromPrivateResponse(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
{
Caster: " Aria ",
Spell: " Cure Wounds ",
SourceRefs: responseSourceRefs(2, 1),
},
}}}
req := extractionRequest()
var prepared promptkit.PreparedRun
req.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value }
extractor := newExtractor(t, client)
result, err := extractor.Extract(context.Background(), req)
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
want := dnd.SpellList{SpellCasts: []dnd.SpellCast{
{
Caster: " Aria ",
Spell: " Cure Wounds ",
SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}},
},
}}
if !reflect.DeepEqual(result.Value, want) {
t.Fatalf("Value = %#v, want %#v", result.Value, want)
}
if len(client.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
}
llmReq := client.requests[0]
if prepared.PromptID != PromptID || prepared.SelectedProfileID != "profile-spells" || prepared.SessionID != "session-123" || llmReq.Target.Model != "spell-test-model" {
t.Fatalf("native selection lost: %#v", prepared)
}
if !strings.Contains(promptText(llmReq), string(req.Chunk.Content)) {
t.Fatal("missing exact chunk bytes")
}
catalogInput := extractor.catalogPromptInput
if !strings.Contains(promptText(llmReq), string(catalogInput)) {
t.Fatal("missing exact catalog projection in native request")
}
var catalogPayload struct {
Spells []spellcatalog.PromptSpell `json:"spells"`
}
if err := json.Unmarshal(catalogInput, &catalogPayload); err != nil {
t.Fatalf("decode catalog prompt input: %v", err)
}
base, err := spellcatalog.LoadSRD5E2014()
if err != nil {
t.Fatal(err)
}
wantNames := make([]string, 0, len(base.Spells()))
for _, spell := range base.Spells() {
wantNames = append(wantNames, spell.Name)
}
sort.Strings(wantNames)
gotNames := make([]string, len(catalogPayload.Spells))
for index, spell := range catalogPayload.Spells {
gotNames[index] = spell.CanonicalName
if !sort.StringsAreSorted(spell.Aliases) {
t.Fatalf("catalog prompt aliases for %q are not sorted: %#v", spell.CanonicalName, spell.Aliases)
}
}
if !reflect.DeepEqual(gotNames, wantNames) || !sort.StringsAreSorted(gotNames) {
t.Fatalf("catalog prompt names = %d entries, want sorted base catalog", len(gotNames))
}
}
func TestExtractPromptProjectsCanonicalNamesAndAliasesWithoutMetadata(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
extractor := newExtractor(t, client, overlaySpellCatalogReference())
if _, err := extractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
input := extractor.catalogPromptInput
content := string(input)
var payload struct {
Spells []spellcatalog.PromptSpell `json:"spells"`
}
if err := json.Unmarshal(input, &payload); err != nil {
t.Fatalf("decode catalog prompt input: %v", err)
}
var aegis *spellcatalog.PromptSpell
for index := range payload.Spells {
if payload.Spells[index].CanonicalName == "Aegis of Emberfall" {
aegis = &payload.Spells[index]
break
}
}
if aegis == nil || !reflect.DeepEqual(aegis.Aliases, []string{"Emberfall Aegis"}) {
t.Fatalf("Aegis prompt projection = %#v, want canonical name and alias", aegis)
}
for _, forbidden := range []string{"Private campaign source", "file:///private-source.json", "private", "license", "ruleset", "provenance"} {
if strings.Contains(content, forbidden) {
t.Fatalf("catalog prompt input leaked %q: %s", forbidden, content)
}
}
}
func TestExtractUsesAliasRecognitionToRequestCanonicalSpellNames(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{{
Caster: "Aria",
Spell: "Aegis of Emberfall",
SourceRefs: responseSourceRefs(1, 1),
}}}}
request := extractionRequest()
request.Chunk.Content = []byte(`{"segments":[{"id":1,"text":"Aria invokes Emberfall Aegis."}]}`)
request.SourceInput = spellChunkInput(request.Chunk)
result, err := newExtractor(t, client, overlaySpellCatalogReference()).Extract(context.Background(), request)
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if got := result.Value.SpellCasts; len(got) != 1 || got[0].Spell != "Aegis of Emberfall" {
t.Fatalf("spell casts = %#v, want canonical spell name", got)
}
content := promptText(client.requests[0])
if !strings.Contains(content, `"aliases":["Emberfall Aegis"]`) {
t.Fatalf("catalog prompt input = %s, want transcript alias recognition", content)
}
}
func TestExtractAliasOnlyCatalogChangesPromptMaterial(t *testing.T) {
aliasReference := spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v1","catalogs":[{"id":"campaign.example","ruleset":"dnd-5e-2014","source":{"title":"Private campaign source","version":"1","url":"file:///private-source.json","license":"private"},"spells":[{"name":"Cure Wounds","aliases":["Campaign Woundweave"]}]}]}`)
baseClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
aliasClient := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
baseExtractor := newExtractor(t, baseClient)
aliasExtractor := newExtractor(t, aliasClient, aliasReference)
if _, err := baseExtractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatalf("base Extract() error = %v", err)
}
if _, err := aliasExtractor.Extract(context.Background(), extractionRequest()); err != nil {
t.Fatalf("alias Extract() error = %v", err)
}
baseInput := promptText(baseClient.requests[0])
aliasInput := promptText(aliasClient.requests[0])
if baseInput == aliasInput || !strings.Contains(aliasInput, "Campaign Woundweave") {
t.Fatal("alias-only catalog change was not reflected in native messages")
}
}
func TestNewRejectsMalformedCatalogBeforeLLMCall(t *testing.T) {
client := &fakeSpellsLLMClient{}
_, err := New(newSpellEngine(t, client), Options{}, spellCatalogReference(`{"schema_version":"notarius.dnd.spell-catalog-overlay.v2","catalogs":[]}`))
if err == nil || !strings.Contains(err.Error(), "resolve effective spell catalog") {
t.Fatalf("New() error = %v, want effective catalog error", err)
}
if len(client.requests) != 0 {
t.Fatalf("LLM calls = %d, want none during failed construction", len(client.requests))
}
}
func TestExtractPassesReferencesAsPromptInputs(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}}
req := extractionRequest()
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Alice: Aria Brightmantle")}}},
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Aria Brightmantle: party cleric")}}},
"glossary": {Slot: contracts.ReferenceSlot{Name: "glossary"}, Items: []contracts.ReferenceItem{{SlotName: "glossary", Content: []byte("Brightmantle: local temple name")}}},
}}
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
text := promptText(client.requests[0])
for _, content := range []string{"Alice: Aria Brightmantle", "Aria Brightmantle: party cleric", "Brightmantle: local temple name", string(req.Chunk.Content)} {
if !strings.Contains(text, content) {
t.Fatalf("missing fixed input %q", content)
}
}
}
func TestPromptInputsMapLegacyRosterReferenceToParty(t *testing.T) {
inputs := shared.PromptInputs(spellSourceInput(), contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
"roster": {Slot: contracts.ReferenceSlot{Name: "roster"}, Items: []contracts.ReferenceItem{{SlotName: "roster", Content: []byte("Legacy roster text")}}},
}})
if got := string(inputs["party"].Content); got != "Legacy roster text" {
t.Fatalf("party input = %q, want legacy roster content", got)
}
if _, ok := inputs["roster"]; ok {
t.Fatal("roster prompt input was present; want only party input")
}
}
func TestExtractPreservesValidEmptyCollection(t *testing.T) {
result, err := newExtractor(t, &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[]}`)}).Extract(context.Background(), extractionRequest())
if err != nil || result.Value.SpellCasts == nil || len(result.Value.SpellCasts) != 0 {
t.Fatalf("empty collection: %#v %v", result, err)
}
}
func TestExtractDoesNotAssessNativeStructuralFailure(t *testing.T) {
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
t.Run(string(mode), func(t *testing.T) {
client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":null}`)}
request := extractionRequest()
request.Operation.Controls.Mode = mode
assessed := 0
request.Operation.Assess = func(context.Context, dnd.SpellList) (contracts.CandidateAssessment, error) {
assessed++
return contracts.CandidateAssessment{}, nil
}
result, err := newExtractor(t, client).Extract(context.Background(), request)
if assessed != 0 {
t.Fatal("structurally invalid native result reached typed validation")
}
if mode == contracts.ModeStrict {
if err == nil {
t.Fatal("strict native structural failure succeeded")
}
} else if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible ||
result.Value.SpellCasts == nil || len(result.Value.SpellCasts) != 0 {
t.Fatalf("permissive native failure = %#v, %v", result, err)
}
})
}
}
func TestExtractWrapsLLMClientError(t *testing.T) {
_, err := newExtractor(t, &fakeSpellsLLMClient{err: errors.New("provider unavailable")}).Extract(context.Background(), extractionRequest())
if err == nil || !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), "provider unavailable") {
t.Fatalf("Extract() error = %v, want wrapped provider error", err)
}
}
func TestExtractRejectsInvalidRequests(t *testing.T) {
validReq := extractionRequest()
validExtractor := newExtractor(t, &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{}}})
var nilExtractor *Extractor
tests := []struct {
name string
extractor *Extractor
ctx context.Context
req contracts.TypedExtractionRequest[dnd.SpellList]
want string
}{
{name: "nil extractor", extractor: nilExtractor, ctx: context.Background(), req: validReq, want: "extractor"},
{name: "nil native engine", extractor: &Extractor{}, ctx: context.Background(), req: validReq, want: "native engine"},
{name: "wrapped preflight failure", extractor: validExtractor, ctx: context.Background(), req: mismatchedSourceInputRequest(validReq), want: "must match chunk"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
_, err := test.extractor.Extract(test.ctx, test.req)
if err == nil || !strings.Contains(err.Error(), "dnd spells") || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Extract() error = %v, want %q context", err, test.want)
}
})
}
}
func TestExtractOrdersAndDeduplicatesEvidence(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
{Caster: "Borin", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(2, 2)},
{Caster: "Aria", Spell: "Cure Wounds", SourceRefs: []spellSourceRefResponse{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 1, EndUnitID: 2}}},
{Caster: "Narrator", Spell: "Unknown", SourceRefs: []spellSourceRefResponse{}},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if got := []string{result.Value.SpellCasts[0].Spell, result.Value.SpellCasts[1].Spell, result.Value.SpellCasts[2].Spell}; !reflect.DeepEqual(got, []string{"Cure Wounds", "Fire Bolt", "Unknown"}) {
t.Fatalf("spell order = %#v, want evidence order", got)
}
if refs := result.Value.SpellCasts[0].SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}) {
t.Fatalf("source refs = %#v, want one canonical ref", refs)
}
}
func TestExtractUsesDocumentOrderForReferencesAndSpellCasts(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
{Caster: "Later", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(10, 10)},
{Caster: "Earlier", Spell: "Cure Wounds", SourceRefs: []spellSourceRefResponse{
{StartUnitID: 10, EndUnitID: 10},
{StartUnitID: 30, EndUnitID: 30},
{StartUnitID: 30, EndUnitID: 30},
{StartUnitID: 999, EndUnitID: 0},
}},
{Caster: "Unavailable", Spell: "Healing Word", SourceRefs: []spellSourceRefResponse{{StartUnitID: 999, EndUnitID: 0}}},
}}}
req := extractionRequest()
req.Source.Units = []source.SourceUnit{{ID: 30}, {ID: 10}}
req.Chunk.Units = append([]source.SourceUnit(nil), req.Source.Units...)
req.Chunk.Ref = source.SourceRef{SourceID: req.Source.ID, StartUnitID: 30, EndUnitID: 10}
result, err := newExtractor(t, client).Extract(context.Background(), req)
if err != nil {
t.Fatalf("Extract() error = %v", err)
}
if got := []string{result.Value.SpellCasts[0].Spell, result.Value.SpellCasts[1].Spell, result.Value.SpellCasts[2].Spell}; !reflect.DeepEqual(got, []string{"Cure Wounds", "Fire Bolt", "Healing Word"}) {
t.Fatalf("spell order = %#v, want document chronology followed by invalid evidence", got)
}
refs := result.Value.SpellCasts[0].SourceRefs
if got := []int{refs[0].StartUnitID, refs[1].StartUnitID, refs[2].StartUnitID}; !reflect.DeepEqual(got, []int{30, 10, 999}) {
t.Fatalf("source refs = %#v, want document order with exact duplicate removed", refs)
}
refs[0].StartUnitID = 777
for _, spell := range client.response.SpellCasts {
for _, ref := range spell.SourceRefs {
if ref.StartUnitID == 777 {
t.Fatal("result source references alias the model response")
}
}
}
}
func TestExtractPreservesStableSpellOrderForEqualEvidence(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{
{Caster: "First", Spell: "Cure Wounds", SourceRefs: responseSourceRefs(2, 2)},
{Caster: "Second", Spell: "Fire Bolt", SourceRefs: responseSourceRefs(2, 2)},
}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v", err)
}
if got := []string{result.Value.SpellCasts[0].Caster, result.Value.SpellCasts[1].Caster}; !reflect.DeepEqual(got, []string{"First", "Second"}) {
t.Fatalf("equal-evidence order = %#v, want stable response order", got)
}
}
func TestExtractPreservesInvalidEvidenceForValidators(t *testing.T) {
client := &fakeSpellsLLMClient{response: extractionResponse{SpellCasts: []spellCastResponse{{
Caster: "Aria", Spell: "Cure Wounds",
SourceRefs: []spellSourceRefResponse{{StartUnitID: 99, EndUnitID: 0}},
}}}}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
ref := result.Value.SpellCasts[0].SourceRefs[0]
if ref != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 99}) {
t.Fatalf("source ref = %#v, want canonical source with invalid range preserved", ref)
}
}
func TestExtractMapsRawSemanticCandidatesWithoutRepair(t *testing.T) {
client := &fakeSpellsLLMClient{content: []byte(`{"spell_casts":[{"caster":"","spell":"Cure Wounds","source_refs":[{"start_unit_id":0,"end_unit_id":-1}]}]}`)}
result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest())
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
spell := result.Value.SpellCasts[0]
if spell.Caster != "" {
t.Fatalf("spell = %#v, want blank semantic values preserved", spell)
}
if refs := spell.SourceRefs; len(refs) != 1 || refs[0] != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 0, EndUnitID: -1}) {
t.Fatalf("source refs = %#v, want raw nonpositive candidates preserved", refs)
}
}