diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ff9086c..5ff1223 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -471,6 +471,12 @@ validation-profile fallback cases and meaningful explicit-value distinctions. ## Stage 11: Derive stage vocabulary and adapt its consumers +**Completed.** Rich context and independent, identity-aware protection views +are available to prompt, validation, and application consumers. The ordered +runner must supply those views and the application check in the next assignment; +older caller tests remain to be reconciled during the assembled cutover. Focused +projection, validator, and application checks passed. + Own the glossary-domain projection and the consumers in `internal/modules`, `internal/validators`, and `internal/framework/proposals`. Use the [derived views](glossary-stages.md#derived-runtime-representations--resolved) diff --git a/internal/core/glossary/selection.go b/internal/core/glossary/selection.go new file mode 100644 index 0000000..02efcd9 --- /dev/null +++ b/internal/core/glossary/selection.go @@ -0,0 +1,39 @@ +// Package glossary derives consumer views from resolved configuration. +package glossary + +import "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + +// Entry is the known glossary content and its collection-scoped identity. +type Entry struct { + Collection string `json:"-"` + ID string `json:"id"` + Name string `json:"name"` + Category string `json:"category"` + Description string `json:"description,omitempty"` + Aliases []string `json:"aliases,omitempty"` + Plural *string `json:"plural,omitempty"` +} + +type identity struct{ collection, id string } + +// Select retains the first occurrence of each selected collection entry. +func Select(collections map[string]config.GlossaryCollection, names []string) []Entry { + seen := make(map[identity]struct{}) + var selected []Entry + for _, name := range names { + for _, item := range collections[name].Entries { + key := identity{collection: name, id: item.ID} + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + entry := Entry{Collection: name, ID: item.ID, Name: item.Name, Category: item.Category, Description: item.Description, Aliases: append([]string(nil), item.Aliases...)} + if item.Plural != nil { + plural := *item.Plural + entry.Plural = &plural + } + selected = append(selected, entry) + } + } + return selected +} diff --git a/internal/core/glossary/selection_test.go b/internal/core/glossary/selection_test.go new file mode 100644 index 0000000..6e5618e --- /dev/null +++ b/internal/core/glossary/selection_test.go @@ -0,0 +1,33 @@ +package glossary + +import ( + "encoding/json" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" +) + +func TestSelectionRetainsDistinctIdentitiesAndDoesNotExposeExtensions(t *testing.T) { + plural := "Shields" + collections := map[string]config.GlossaryCollection{ + "spells": {Entries: []config.GlossaryEntry{{ID: "shield", Name: "Shield", Category: "spell", Aliases: []string{"Ward"}, Plural: &plural, Extra: map[string]any{"private": "do not forward"}}}}, + "items": {Entries: []config.GlossaryEntry{{ID: "shield", Name: "Shield", Category: "item"}}}, + } + result := Select(collections, []string{"spells", "items", "spells"}) + if len(result) != 2 || result[0].Collection != "spells" || result[1].Collection != "items" { + t.Fatalf("entries=%+v", result) + } + raw, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "do not forward") { + t.Fatal("extension leaked into prompt projection") + } + result[0].Aliases[0] = "changed" + *result[0].Plural = "changed" + if collections["spells"].Entries[0].Aliases[0] != "Ward" || *collections["spells"].Entries[0].Plural != "Shields" { + t.Fatal("projection changed configuration") + } +} diff --git a/internal/core/glossary/vocabulary.go b/internal/core/glossary/vocabulary.go new file mode 100644 index 0000000..a83f2e3 --- /dev/null +++ b/internal/core/glossary/vocabulary.go @@ -0,0 +1,71 @@ +package glossary + +import "strings" + +// Identity identifies an entry within its configured collection. +type Identity struct { + Collection string + ID string +} + +// Term retains every entry that contributes a recognized spelling. +type Term struct { + Spelling string + Identities []Identity +} + +// Vocabulary derives recognized spellings in selection order. Excluded entries +// remove matching spellings regardless of which protected entries supplied them. +func Vocabulary(entries, excluded []Entry) []Term { + exclusions := make(map[string]struct{}) + for _, entry := range excluded { + for _, spelling := range spellings(entry) { + exclusions[strings.ToLower(spelling)] = struct{}{} + } + } + indices := make(map[string]int) + var terms []Term + for _, entry := range entries { + id := Identity{Collection: entry.Collection, ID: entry.ID} + for _, spelling := range spellings(entry) { + key := strings.ToLower(spelling) + if _, excluded := exclusions[key]; excluded { + continue + } + index, exists := indices[key] + if !exists { + index = len(terms) + indices[key] = index + terms = append(terms, Term{Spelling: spelling}) + } + identities := terms[index].Identities + found := false + for _, existing := range identities { + if existing == id { + found = true + break + } + } + if !found { + terms[index].Identities = append(identities, id) + } + } + } + return terms +} + +func spellings(entry Entry) []string { + var result []string + for _, value := range append([]string{entry.Name}, entry.Aliases...) { + value = strings.TrimSpace(value) + if value != "" { + result = append(result, value, value+"s") + } + } + if entry.Plural != nil { + if plural := strings.TrimSpace(*entry.Plural); plural != "" { + result = append(result, plural) + } + } + return result +} diff --git a/internal/core/glossary/vocabulary_test.go b/internal/core/glossary/vocabulary_test.go new file mode 100644 index 0000000..d406f7b --- /dev/null +++ b/internal/core/glossary/vocabulary_test.go @@ -0,0 +1,30 @@ +package glossary + +import ( + "reflect" + "testing" +) + +func TestVocabularyKeepsContributorsAndExemptsOnlyActiveSpellings(t *testing.T) { + plural := "Aegides" + active := []Entry{{Collection: "spells", ID: "shield", Name: "Shield", Aliases: []string{"Ward"}}} + protected := []Entry{ + active[0], + {Collection: "items", ID: "shield", Name: "shield", Aliases: []string{"Aegis"}, Plural: &plural}, + } + full := Vocabulary(protected, nil) + if !reflect.DeepEqual(full[0].Identities, []Identity{{Collection: "spells", ID: "shield"}, {Collection: "items", ID: "shield"}}) { + t.Fatalf("shared spelling contributors: %+v", full[0]) + } + reduced := Vocabulary(protected, active) + var got []string + for _, term := range reduced { + got = append(got, term.Spelling) + } + if !reflect.DeepEqual(got, []string{"Aegis", "Aegiss", "Aegides"}) { + t.Fatalf("remaining spellings: %v", got) + } + if !reflect.DeepEqual(full, Vocabulary(protected, nil)) || protected[0].Aliases[0] != "Ward" { + t.Fatal("deriving an exemption changed another view or its inputs") + } +} diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index a359f43..851b655 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -6,6 +6,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -65,7 +66,8 @@ type ExecutionContext struct { ReportDiagnosticError func(error) `json:"-"` Config *config.Config `json:"-"` WorkingTranscript *schema.Transcript `json:"-"` - Glossary *schema.Glossary `json:"-"` + Glossary []glossary.Entry `json:"-"` + ProtectedTerms []glossary.Term `json:"-"` Section *SectionMetadata `json:"section,omitempty"` DiagnosticsDir string `json:"diagnostics_dir,omitempty"` } diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 1c9aaf9..87fe99c 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -57,7 +57,6 @@ type Request struct { ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` WorkingTranscript *schema.Transcript `json:"-"` Section *contracts.SectionMetadata `json:"section,omitempty"` - Glossary *schema.Glossary `json:"-"` Config *config.Config `json:"-"` PromptID string `json:"prompt_id"` Variables map[string]string `json:"variables"` diff --git a/internal/framework/proposal_generation/glossary_projection_test.go b/internal/framework/proposal_generation/glossary_projection_test.go new file mode 100644 index 0000000..97db8a8 --- /dev/null +++ b/internal/framework/proposal_generation/glossary_projection_test.go @@ -0,0 +1,27 @@ +package proposal_generation + +import ( + "encoding/json" + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" +) + +func TestProposalContextPreservesDistinctEntries(t *testing.T) { + entries := []glossary.Entry{ + {Collection: "spells", ID: "shield", Name: "Shield", Category: "spell", Description: "A spell"}, + {Collection: "items", ID: "shield", Name: "Shield", Category: "item", Description: "An item"}, + } + variables, err := BuildProposalVariables(&schema.Transcript{}, entries, 0, "") + if err != nil { + t.Fatal(err) + } + var decoded []glossary.Entry + if err := json.Unmarshal([]byte(variables["GlossaryJSON"]), &decoded); err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Category != "spell" || decoded[1].Category != "item" { + t.Fatalf("prompt entries=%+v", decoded) + } +} diff --git a/internal/framework/proposal_generation/module_proposal.go b/internal/framework/proposal_generation/module_proposal.go index 775ae1c..eadd679 100644 --- a/internal/framework/proposal_generation/module_proposal.go +++ b/internal/framework/proposal_generation/module_proposal.go @@ -5,13 +5,14 @@ import ( "fmt" "strings" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" ) -type ProposalVariableBuilder func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) +type ProposalVariableBuilder func(transcript *schema.Transcript, entries []glossary.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) type ModuleProposalRequest struct { ProposalRequest contracts.ProposalRequest @@ -60,7 +61,6 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy, WorkingTranscript: req.ProposalRequest.WorkingTranscript, Section: req.ProposalRequest.Section, - Glossary: req.ProposalRequest.Glossary, Config: req.ProposalRequest.Config, PromptID: promptMetadata.PromptID, Variables: variables, diff --git a/internal/framework/proposal_generation/variables.go b/internal/framework/proposal_generation/variables.go index c7508c2..53dbcec 100644 --- a/internal/framework/proposal_generation/variables.go +++ b/internal/framework/proposal_generation/variables.go @@ -4,14 +4,15 @@ import ( "encoding/json" "fmt" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext" ) // BuildProposalVariables preserves the domain inputs while PromptKit renders // the selected embedded proposal definition. -func BuildProposalVariables(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) { - glossaryJSON, err := json.MarshalIndent(glossary, "", " ") +func BuildProposalVariables(transcript *schema.Transcript, entries []glossary.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) { + glossaryJSON, err := json.MarshalIndent(entries, "", " ") if err != nil { return nil, fmt.Errorf("marshal glossary prompt context: %w", err) } diff --git a/internal/framework/proposals/apply.go b/internal/framework/proposals/apply.go index d0341df..7219439 100644 --- a/internal/framework/proposals/apply.go +++ b/internal/framework/proposals/apply.go @@ -43,6 +43,7 @@ func ApplyProposals( transcript *schema.Transcript, proposals []EnrichedCorrectionProposal, policy ReplacementPolicy, + allowsChange func(before, after string) bool, ) ApplyResult { working := cloneTranscript(transcript) ordered := sortProposalsByIndex(proposals) @@ -68,6 +69,11 @@ func ApplyProposals( continue } + if allowsChange != nil && !allowsChange(working.Segments[segmentIdx].Text, preview.CorrectedSegmentText) { + skipped = append(skipped, newSkippedChange(proposal, SkipReasonProtectedTerm)) + continue + } + working.Segments[segmentIdx].Text = preview.CorrectedSegmentText applied = append(applied, AppliedChange{ ProposalIndex: proposal.ProposalIndex, @@ -149,6 +155,8 @@ func skipReasonMessage(reason ProposalSkipReason) string { return "original_text and corrected_text are identical" case SkipReasonEmptyResultingText: return "proposal would leave the segment empty" + case SkipReasonProtectedTerm: + return "proposal may alter protected glossary terminology" case SkipReasonInvalidProposal: return "proposal failed structural or policy validation" default: diff --git a/internal/framework/proposals/apply_fixtures_test.go b/internal/framework/proposals/apply_fixtures_test.go index 3ab8879..f48d24c 100644 --- a/internal/framework/proposals/apply_fixtures_test.go +++ b/internal/framework/proposals/apply_fixtures_test.go @@ -38,7 +38,7 @@ func TestApplyProposalsFixtureTranscriptGoldens(t *testing.T) { t.Fatalf("failed to parse fixture policy: %v", err) } - result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy) + result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy, nil) assertJSONSemanticallyEqualFromGolden(t, result.Transcript, name+".transcript.golden.json") }) } @@ -62,7 +62,7 @@ func TestApplyProposalsFixtureSkipCoverage(t *testing.T) { t.Fatalf("failed to parse fixture policy: %v", err) } - result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy) + result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy, nil) if len(result.Skipped) == 0 { t.Fatal("expected at least one skipped change") } @@ -81,7 +81,7 @@ func TestApplyProposalsFixtureMixedScenarioRecordGoldens(t *testing.T) { t.Fatalf("failed to parse fixture policy: %v", err) } - result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy) + result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy, nil) payload := recordsPayload{ AppliedChanges: result.Applied, SkippedChanges: result.Skipped, diff --git a/internal/framework/proposals/apply_test.go b/internal/framework/proposals/apply_test.go index 610cf62..8d29d65 100644 --- a/internal/framework/proposals/apply_test.go +++ b/internal/framework/proposals/apply_test.go @@ -21,7 +21,7 @@ func TestApplyProposalsApplyingOneProposal(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique) + result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil) if got := result.Transcript.Segments[0].Text; got != "Jesters" { t.Fatalf("expected corrected text Jesters, got %q", got) @@ -57,7 +57,7 @@ func TestApplyProposalsApplyingMultipleProposalsInStableOrder(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll) + result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll, nil) if got := result.Transcript.Segments[0].Text; got != "X X" { t.Fatalf("expected final text X X, got %q", got) @@ -93,7 +93,7 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique) + result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil) if got := result.Transcript.Segments[0].Text; got != "foo-bar" { t.Fatalf("expected final text foo-bar, got %q", got) @@ -120,7 +120,7 @@ func TestApplyProposalsRechecksBlankResultAfterEarlierDeletion(t *testing.T) { t.Fatalf("proposal must be initially applicable: %+v", preview) } } - result := ApplyProposals(transcript, proposed, ReplacementPolicyRequireUnique) + result := ApplyProposals(transcript, proposed, ReplacementPolicyRequireUnique, nil) if got := result.Transcript.Segments[0].Text; got != " b" { t.Fatalf("expected literal surviving text, got %q", got) } @@ -143,7 +143,7 @@ func TestApplyProposalsMissingSegmentSkip(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique) + result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil) if len(result.Skipped) != 1 { t.Fatalf("expected 1 skipped change, got %d", len(result.Skipped)) @@ -167,7 +167,7 @@ func TestApplyProposalsRequireUniqueAmbiguousSkip(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique) + result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil) if len(result.Skipped) != 1 { t.Fatalf("expected 1 skipped change, got %d", len(result.Skipped)) @@ -191,7 +191,7 @@ func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll) + result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll, nil) if len(result.Applied) != 1 { t.Fatalf("expected 1 applied change, got %d", len(result.Applied)) @@ -220,7 +220,7 @@ func TestApplyProposalsNoMutationOfInputTranscript(t *testing.T) { }, } - _ = ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique) + _ = ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil) if !reflect.DeepEqual(transcript, before) { t.Fatalf("input transcript mutated: before=%+v after=%+v", before, transcript) @@ -249,7 +249,7 @@ func TestApplyProposalsSegmentMetadataPreservation(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique) + result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil) seg := result.Transcript.Segments[0] if seg.Speaker != "A" || seg.Start != 12.5 || seg.End != 15.0 { @@ -283,7 +283,7 @@ func TestApplyProposalsAppliedAndSkippedRecords(t *testing.T) { }, } - result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique) + result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil) if len(result.Applied) != 1 || len(result.Skipped) != 1 { t.Fatalf("expected 1 applied and 1 skipped, got applied=%d skipped=%d", len(result.Applied), len(result.Skipped)) diff --git a/internal/framework/proposals/preview.go b/internal/framework/proposals/preview.go index 89c36c1..6f2d8c6 100644 --- a/internal/framework/proposals/preview.go +++ b/internal/framework/proposals/preview.go @@ -17,6 +17,7 @@ const ( SkipReasonNoEffect ProposalSkipReason = "no_effect" SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment" SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal" + SkipReasonProtectedTerm ProposalSkipReason = "protected_glossary_term" ) // SegmentPreviewResult is a deterministic preview of applying one proposal to diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index cd048e1..2c7855a 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -136,17 +136,12 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err return Result{ValidatorName: v.Name(), Decisions: decisions}, nil } -func ValidateProtectedTerms(req Request, glossaryStage bool) (Result, error) { - vocab := NewProtectedVocabulary(req.Glossary) +func ValidateProtectedTerms(req Request) (Result, error) { + vocab := NewProtectedVocabulary(req.ProtectedTerms) decisions := make([]Decision, 0, len(req.CandidateProposal)) for _, c := range req.CandidateProposal { - var reason string - if glossaryStage { - reason = vocab.glossaryStageViolationReason(c.OriginalText, c.CorrectedText) - } else { - reason = vocab.violationReason(c.OriginalText, c.CorrectedText) - } - if reason != "" && !glossaryStage && req.ModuleKey == modulecatalog.KeySpokenWord && req.WorkingTranscript != nil { + reason := vocab.violationReason(c.OriginalText, c.CorrectedText) + if reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord && req.WorkingTranscript != nil { for i := range req.WorkingTranscript.Segments { segment := &req.WorkingTranscript.Segments[i] if segment.ID != c.TargetSegmentID { diff --git a/internal/framework/validators/llm_context.go b/internal/framework/validators/llm_context.go index f27eeb9..9d631ba 100644 --- a/internal/framework/validators/llm_context.go +++ b/internal/framework/validators/llm_context.go @@ -2,13 +2,14 @@ package validators import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) // validationContext supplies adjacent source segments within the current chunk // and glossary definitions mentioned in the candidate or that local context. // These fields travel in each item so batching accounts for their size. -func validationContext(segments []schema.Segment, position int, glossary *schema.Glossary, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) { +func validationContext(segments []schema.Segment, position int, selected []glossary.Entry, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []glossary.Entry) { var neighbors []NeighboringSegment source := segments[position].Text if position > 0 { @@ -22,16 +23,16 @@ func validationContext(segments []schema.Segment, position int, glossary *schema for _, neighbor := range neighbors { texts = append(texts, neighbor.Text) } - var entries []schema.GlossaryEntry - if glossary != nil { - mentioned := make(map[int]bool) + var entries []glossary.Entry + if selected != nil { + mentioned := make(map[glossary.Identity]bool) for _, text := range texts { for identity := range vocabulary.occurrencesByIdentity(text) { mentioned[identity] = true } } - for i, entry := range glossary.Entries { - if mentioned[i] { + for _, entry := range selected { + if mentioned[glossary.Identity{Collection: entry.Collection, ID: entry.ID}] { entries = append(entries, entry) } } diff --git a/internal/framework/validators/llm_models.go b/internal/framework/validators/llm_models.go index 4f74579..841b238 100644 --- a/internal/framework/validators/llm_models.go +++ b/internal/framework/validators/llm_models.go @@ -1,7 +1,7 @@ package validators import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) @@ -20,15 +20,15 @@ type StructuredLLMClient = structuredoutput.Client // LLMValidationItem retains the internal proposal index until prompt encoding. // BuildValidationVariables substitutes a one-based reference for each batch. type LLMValidationItem struct { - CorrectionIndex int `json:"correction_index"` - SegmentID int `json:"-"` - OriginalText string `json:"original_text"` - CorrectedText string `json:"corrected_text"` - OriginalSegmentText string `json:"original_segment_text"` - CorrectedSegmentText string `json:"corrected_segment_text"` - Categories []string `json:"categories,omitempty"` - NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"` - GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"` + CorrectionIndex int `json:"correction_index"` + SegmentID int `json:"-"` + OriginalText string `json:"original_text"` + CorrectedText string `json:"corrected_text"` + OriginalSegmentText string `json:"original_segment_text"` + CorrectedSegmentText string `json:"corrected_segment_text"` + Categories []string `json:"categories,omitempty"` + NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"` + GlossaryEntries []glossary.Entry `json:"glossary_entries,omitempty"` } // LLMValidationRequest is the canonical request model for LLM-backed validators. @@ -38,7 +38,6 @@ type LLMValidationRequest struct { ModuleKey string `json:"module_key"` ModuleInstance string `json:"module_instance"` ReplacementPolicy string `json:"replacement_policy"` - Glossary *schema.Glossary `json:"-"` Items []LLMValidationItem `json:"corrections"` } diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index ddf17a6..1668d63 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -9,6 +9,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" @@ -253,7 +254,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT immediate := make([]Decision, 0) var segments []schema.Segment positions := make(map[int]int) - vocabulary := NewProtectedVocabulary(req.Glossary) + vocabulary := NewProtectedVocabulary(glossary.Vocabulary(req.Glossary, nil)) if req.WorkingTranscript != nil { segments = req.WorkingTranscript.Segments for i, seg := range segments { @@ -295,7 +296,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, ReplacementPolicy: string(req.ReplacementPolicy), - Glossary: req.Glossary, Items: items, }, immediate } diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index ba9bbaf..b874dd8 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -6,6 +6,7 @@ import ( "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" @@ -41,7 +42,8 @@ type Request struct { ModuleInstance string `json:"module_instance"` ValidatorOrder *int `json:"validator_order,omitempty"` ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` - Glossary *schema.Glossary `json:"-"` + Glossary []glossary.Entry `json:"-"` + ProtectedTerms []glossary.Term `json:"-"` Config *config.Config `json:"-"` LLMClient StructuredLLMClient `json:"-"` DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"` diff --git a/internal/framework/validators/protected_terms.go b/internal/framework/validators/protected_terms.go index 503ee85..41b40e4 100644 --- a/internal/framework/validators/protected_terms.go +++ b/internal/framework/validators/protected_terms.go @@ -5,17 +5,17 @@ import ( "sort" "strings" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" ) type protectedTermDef struct { - identity int - canonical string + identities []glossary.Identity + canonical string } type protectedOccurrence struct { text string - identity int + identity glossary.Identity canonical string } @@ -25,47 +25,13 @@ type ProtectedVocabulary struct { pattern *regexp.Regexp } -// ExtractProtectedTerms returns a stable, de-duplicated list of protected terms -// derived from glossary names, aliases, synthetic plural forms, and explicit -// plural fields. -func ExtractProtectedTerms(glossary *schema.Glossary) []string { - vocab := NewProtectedVocabulary(glossary) - out := make([]string, 0, len(vocab.termsByFolded)) - for _, def := range vocab.termsByFolded { - out = append(out, def.canonical) - } - sort.SliceStable(out, func(i, j int) bool { - li := strings.ToLower(out[i]) - lj := strings.ToLower(out[j]) - if li == lj { - return out[i] < out[j] - } - return li < lj - }) - return out -} - -// NewProtectedVocabulary builds a deterministic protected vocabulary from a glossary. -func NewProtectedVocabulary(glossary *schema.Glossary) ProtectedVocabulary { - termsByFolded := make(map[string]protectedTermDef) - if glossary != nil { - for identity, entry := range glossary.Entries { - entryTerms := append([]string{entry.Name}, entry.Aliases...) - for _, term := range entryTerms { - trimmed := strings.TrimSpace(term) - if trimmed == "" { - continue - } - addProtectedTerm(termsByFolded, trimmed, identity) - addProtectedTerm(termsByFolded, trimmed+"s", identity) - } - addProtectedTerm(termsByFolded, entry.Plural, identity) - } - } - - alternatives := make([]string, 0, len(termsByFolded)) - for _, def := range termsByFolded { - alternatives = append(alternatives, regexp.QuoteMeta(def.canonical)) +// NewProtectedVocabulary prepares the independently selected terms for matching. +func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary { + termsByFolded := make(map[string]protectedTermDef, len(terms)) + alternatives := make([]string, 0, len(terms)) + for _, term := range terms { + termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling} + alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling)) } sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) }) if len(alternatives) == 0 { @@ -75,16 +41,13 @@ func NewProtectedVocabulary(glossary *schema.Glossary) ProtectedVocabulary { return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern} } -func addProtectedTerm(terms map[string]protectedTermDef, term string, identity int) { - trimmed := strings.TrimSpace(term) - if trimmed == "" { - return +// AllowsChange checks a proposed segment against the effective protection view. +// Spoken-word cleanup may reduce repeated mentions while preserving each identity. +func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool { + if v.violationReason(before, after) == "" { + return true } - folded := strings.ToLower(trimmed) - if _, ok := terms[folded]; ok { - return - } - terms[folded] = protectedTermDef{identity: identity, canonical: trimmed} + return allowRepetitionCleanup && v.preservesRepeatedIdentities(before, after) } func (v ProtectedVocabulary) violationReason(before, after string) string { @@ -99,20 +62,8 @@ func (v ProtectedVocabulary) violationReason(before, after string) string { return "" } -func (v ProtectedVocabulary) glossaryStageViolationReason(before, after string) string { - beforeByID := v.occurrencesByIdentity(before) - afterByID := v.occurrencesByIdentity(after) - if reason := validateGlossaryStageIdentityPreservation(beforeByID, afterByID); reason != "" { - return reason - } - if reason := validateCapitalizationTransitions(beforeByID, afterByID); reason != "" { - return reason - } - return "" -} - -func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[int][]protectedOccurrence { - byID := make(map[int][]protectedOccurrence) +func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[glossary.Identity][]protectedOccurrence { + byID := make(map[glossary.Identity][]protectedOccurrence) for _, o := range v.occurrences(text) { byID[o.identity] = append(byID[o.identity], o) } @@ -134,16 +85,14 @@ func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence { if !ok { continue } - out = append(out, protectedOccurrence{ - text: matched, - identity: def.identity, - canonical: def.canonical, - }) + for _, identity := range def.identities { + out = append(out, protectedOccurrence{text: matched, identity: identity, canonical: def.canonical}) + } } return out } -func validateIdentityPreservation(beforeByID, afterByID map[int][]protectedOccurrence) string { +func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string { for identity, beforeItems := range beforeByID { if len(afterByID[identity]) < len(beforeItems) { return "proposal may alter protected glossary terminology" @@ -152,22 +101,7 @@ func validateIdentityPreservation(beforeByID, afterByID map[int][]protectedOccur return "" } -func validateGlossaryStageIdentityPreservation(beforeByID, afterByID map[int][]protectedOccurrence) string { - beforeTotal := 0 - for _, items := range beforeByID { - beforeTotal += len(items) - } - afterTotal := 0 - for _, items := range afterByID { - afterTotal += len(items) - } - if afterTotal < beforeTotal { - return "proposal may alter protected glossary terminology" - } - return "" -} - -func validateCapitalizationTransitions(beforeByID, afterByID map[int][]protectedOccurrence) string { +func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string { for identity, afterItems := range afterByID { beforeItems := beforeByID[identity] for i, afterItem := range afterItems { diff --git a/internal/framework/validators/vocabulary_test.go b/internal/framework/validators/vocabulary_test.go new file mode 100644 index 0000000..2cdb7ce --- /dev/null +++ b/internal/framework/validators/vocabulary_test.go @@ -0,0 +1,75 @@ +package validators + +import ( + "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" +) + +func TestIndependentProtectionForSelectedGlossary(t *testing.T) { + active := []glossary.Entry{{Collection: "spells", ID: "shield", Name: "Shield"}} + entries := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "shield", Name: "Shield", Aliases: []string{"Aegis"}}) + cases := []struct { + name string + terms []glossary.Term + before, after string + approved bool + }{ + {"shared spelling exempt", glossary.Vocabulary(entries, active), "Shield", "Barrier", true}, + {"other alias protected", glossary.Vocabulary(entries, active), "Aegis", "Armor", false}, + {"full protection retained", glossary.Vocabulary(entries, nil), "Shield", "Barrier", false}, + {"empty protection", nil, "Aegis", "Armor", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + result, err := ValidateProtectedTerms(Request{ProtectedTerms: tc.terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{OriginalText: tc.before, CorrectedText: tc.after}}}}) + if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.approved { + t.Fatalf("result=%+v err=%v", result, err) + } + }) + } +} + +func TestSharedAliasRetainsAllIdentities(t *testing.T) { + entries := []glossary.Entry{ + {Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}}, + {Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}}, + } + vocabulary := NewProtectedVocabulary(glossary.Vocabulary(entries, nil)) + identities := vocabulary.occurrencesByIdentity("Shared") + if len(identities) != 2 || len(identities[glossary.Identity{Collection: "a", ID: "first"}]) != 1 || len(identities[glossary.Identity{Collection: "b", ID: "second"}]) != 1 { + t.Fatalf("identities=%v", identities) + } +} + +func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) { + active := []glossary.Entry{{Collection: "spells", ID: "spell", Name: "Shield"}} + protected := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "item", Name: "Shield", Aliases: []string{"Aegis"}}) + vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active)) + transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}} + candidates := []proposals.EnrichedCorrectionProposal{ + {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}, + {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}}, + } + result := proposals.ApplyProposals(transcript, candidates, proposals.ReplacementPolicyRequireUnique, func(before, after string) bool { + return vocabulary.AllowsChange(before, after, false) + }) + if result.Transcript.Segments[0].Text != "Barrier and Aegis" || len(result.Applied) != 1 || len(result.Skipped) != 1 || result.Skipped[0].SkipReason != proposals.SkipReasonProtectedTerm { + t.Fatalf("unexpected application: %+v", result) + } +} + +func TestValidationContextIncludesEachSharedDefinition(t *testing.T) { + entries := []glossary.Entry{ + {Collection: "spells", ID: "shield", Name: "Shield", Category: "spell"}, + {Collection: "items", ID: "shield", Name: "Shield", Category: "item"}, + {Collection: "items", ID: "sword", Name: "Sword", Category: "item"}, + } + segments := []schema.Segment{{ID: 1, Text: "Shield"}} + _, selected := validationContext(segments, 0, entries, NewProtectedVocabulary(glossary.Vocabulary(entries, nil)), "Barrier") + if len(selected) != 2 || selected[0].Category != "spell" || selected[1].ID != "shield" || selected[1].Collection != "items" { + t.Fatalf("selected=%+v", selected) + } +} diff --git a/internal/validators/chains.go b/internal/validators/chains.go index e043aaa..9ed0177 100644 --- a/internal/validators/chains.go +++ b/internal/validators/chains.go @@ -6,7 +6,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" - "gitea.maximumdirect.net/eric/audita/v3/internal/validators/protected_terms" ) var builtInChains = map[string][]string{ @@ -75,16 +74,6 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali out := make([]contracts.Validator, 0, len(keys)) for _, key := range keys { - if moduleKey == modulecatalog.KeyGlossary && key == KeyProtectedTerms { - // Glossary stages preserve current stricter protection semantics while - // reporting the stable protected_terms key. - v, buildErr := protected_terms.NewGlossaryStage() - if buildErr != nil { - return nil, buildErr - } - out = append(out, v) - continue - } v, buildErr := registry.MustBuild(key) if buildErr != nil { return nil, buildErr diff --git a/internal/validators/protected_terms/validator.go b/internal/validators/protected_terms/validator.go index 2a60fa3..e8dcd05 100644 --- a/internal/validators/protected_terms/validator.go +++ b/internal/validators/protected_terms/validator.go @@ -14,31 +14,10 @@ func New() (contracts.Validator, error) { return validatormetadata.Wrap(validator{}, validatormetadata.ExecutionClassDeterministic), nil } -func NewGlossaryStage() (contracts.Validator, error) { - return validatormetadata.Wrap(validator{glossaryStage: true}, validatormetadata.ExecutionClassDeterministic), nil -} +type validator struct{} -type validator struct { - glossaryStage bool -} - -func (v validator) Name() string { - return Key -} +func (v validator) Name() string { return Key } func (v validator) Validate(_ context.Context, req contracts.ValidationRequest) (frameworkvalidators.Result, error) { - if !v.glossaryStage && req.ModuleKey == "glossary" { - decisions := make([]frameworkvalidators.Decision, 0, len(req.CandidateProposal)) - for _, c := range req.CandidateProposal { - decisions = append(decisions, frameworkvalidators.Decision{ - ProposalIndex: c.ProposalIndex, - Approved: true, - ReasonCode: frameworkvalidators.ReasonApproved, - Message: "approved", - }) - } - return frameworkvalidators.Result{ValidatorName: Key, Decisions: decisions}, nil - } - - return frameworkvalidators.ValidateProtectedTerms(req, v.glossaryStage) + return frameworkvalidators.ValidateProtectedTerms(req) }