Index source positions during chunk plan validation
This commit is contained in:
@@ -319,6 +319,11 @@ run contracts, pipeline, CLI, and module integration tests after removal.
|
||||
|
||||
## Finding 6: Use Indexed Source Validation In The Chunk-Plan Hot Path
|
||||
|
||||
**Status: Complete.** Document validation builds one unit-position map, and plan
|
||||
materialization reuses checked endpoints without rescanning or duplicate
|
||||
framework validation. Source, chunk-map, and pipeline tests cover retained
|
||||
ordering, ownership, malformed-input behavior, and nonmonotonic IDs.
|
||||
|
||||
Priority: P2. Relevant stages: 16, 24, 28–29, and 37.
|
||||
|
||||
### Evidence And Complexity
|
||||
|
||||
@@ -117,51 +117,63 @@ func CanonicalizeChunkPlan(plan ChunkPlan) (ChunkPlan, error) {
|
||||
// ValidateChunkPlan validates a canonical plan against the current source.
|
||||
// Ranges may contain gaps or overlap, but their start positions must increase.
|
||||
func ValidateChunkPlan(doc *SourceDocument, plan ChunkPlan) error {
|
||||
if err := ValidateDocument(doc); err != nil {
|
||||
return fmt.Errorf("source document: %w", err)
|
||||
_, err := validateChunkPlan(doc, plan)
|
||||
return err
|
||||
}
|
||||
|
||||
type chunkPositions struct{ start, end int }
|
||||
|
||||
// Retain resolved positions so materialization does not rescan the document or
|
||||
// repeat validation. Ordering is by document position, never numeric unit ID.
|
||||
func validateChunkPlan(doc *SourceDocument, plan ChunkPlan) ([]chunkPositions, error) {
|
||||
index, err := validateDocument(doc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source document: %w", err)
|
||||
}
|
||||
if plan.SourceDigest != doc.Digest {
|
||||
return fmt.Errorf("chunk plan source_digest %q does not match source document digest %q", plan.SourceDigest, doc.Digest)
|
||||
return nil, fmt.Errorf("chunk plan source_digest %q does not match source document digest %q", plan.SourceDigest, doc.Digest)
|
||||
}
|
||||
if len(plan.Ranges) == 0 {
|
||||
return fmt.Errorf("chunk plan ranges must not be empty")
|
||||
return nil, fmt.Errorf("chunk plan ranges must not be empty")
|
||||
}
|
||||
if err := ValidateChunkAnnotations(plan.Annotations); err != nil {
|
||||
return fmt.Errorf("chunk plan annotations: %w", err)
|
||||
return nil, fmt.Errorf("chunk plan annotations: %w", err)
|
||||
}
|
||||
positions := make([]chunkPositions, len(plan.Ranges))
|
||||
previousStart := -1
|
||||
for i, chunkRange := range plan.Ranges {
|
||||
start, ok := UnitIndex(doc, chunkRange.StartUnitID)
|
||||
start, ok := index.Position(chunkRange.StartUnitID)
|
||||
if !ok {
|
||||
return fmt.Errorf("chunk plan range[%d] start_unit_id %d was not found", i, chunkRange.StartUnitID)
|
||||
return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d was not found", i, chunkRange.StartUnitID)
|
||||
}
|
||||
end, ok := UnitIndex(doc, chunkRange.EndUnitID)
|
||||
end, ok := index.Position(chunkRange.EndUnitID)
|
||||
if !ok {
|
||||
return fmt.Errorf("chunk plan range[%d] end_unit_id %d was not found", i, chunkRange.EndUnitID)
|
||||
return nil, fmt.Errorf("chunk plan range[%d] end_unit_id %d was not found", i, chunkRange.EndUnitID)
|
||||
}
|
||||
if start > end {
|
||||
return fmt.Errorf("chunk plan range[%d] start_unit_id %d appears after end_unit_id %d", i, chunkRange.StartUnitID, chunkRange.EndUnitID)
|
||||
return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d appears after end_unit_id %d", i, chunkRange.StartUnitID, chunkRange.EndUnitID)
|
||||
}
|
||||
if start <= previousStart {
|
||||
return fmt.Errorf("chunk plan range[%d] start_unit_id %d does not appear after the previous range start", i, chunkRange.StartUnitID)
|
||||
return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d does not appear after the previous range start", i, chunkRange.StartUnitID)
|
||||
}
|
||||
if err := ValidateChunkAnnotations(chunkRange.Annotations); err != nil {
|
||||
return fmt.Errorf("chunk plan range[%d] annotations: %w", i, err)
|
||||
return nil, fmt.Errorf("chunk plan range[%d] annotations: %w", i, err)
|
||||
}
|
||||
positions[i] = chunkPositions{start: start, end: end}
|
||||
previousStart = start
|
||||
}
|
||||
return nil
|
||||
return positions, nil
|
||||
}
|
||||
|
||||
// MaterializeChunkPlan deterministically expands a validated plan into chunks.
|
||||
func MaterializeChunkPlan(doc *SourceDocument, plan ChunkPlan) ([]Chunk, error) {
|
||||
if err := ValidateChunkPlan(doc, plan); err != nil {
|
||||
positions, err := validateChunkPlan(doc, plan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chunks := make([]Chunk, 0, len(plan.Ranges))
|
||||
for index, chunkRange := range plan.Ranges {
|
||||
start, _ := UnitIndex(doc, chunkRange.StartUnitID)
|
||||
end, _ := UnitIndex(doc, chunkRange.EndUnitID)
|
||||
start, end := positions[index].start, positions[index].end
|
||||
units, err := cloneSourceUnits(doc.Units[start : end+1])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("clone chunk plan range[%d] units: %w", index, err)
|
||||
|
||||
@@ -312,6 +312,33 @@ func planDocument() *SourceDocument {
|
||||
return doc
|
||||
}
|
||||
|
||||
func TestChunkPlanUsesDocumentOrderForNonmonotonicUnitIDs(t *testing.T) {
|
||||
doc := planDocument()
|
||||
doc.Units = doc.Units[:3]
|
||||
for i, id := range []int{90, 7, 42} {
|
||||
doc.Units[i].ID = id
|
||||
doc.Units[i].Ref = SourceRef{SourceID: doc.ID, StartUnitID: id, EndUnitID: id}
|
||||
}
|
||||
plan := ChunkPlan{SourceDigest: doc.Digest, Ranges: []ChunkRange{{StartUnitID: 90, EndUnitID: 42}, {StartUnitID: 7, EndUnitID: 42}}}
|
||||
if err := ValidateChunkPlan(doc, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chunks, err := MaterializeChunkPlan(doc, plan)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{90, 7, 42}) || !reflect.DeepEqual(unitIDs(chunks[1].Units), []int{7, 42}) {
|
||||
t.Fatalf("nonmonotonic IDs changed materialization order: %#v", chunks)
|
||||
}
|
||||
plan.Ranges = []ChunkRange{{StartUnitID: 42, EndUnitID: 90}}
|
||||
if err := ValidateChunkPlan(doc, plan); err == nil {
|
||||
t.Fatal("reversed document positions passed validation")
|
||||
}
|
||||
if _, err := MaterializeChunkPlan(doc, plan); err == nil {
|
||||
t.Fatal("reversed document positions materialized")
|
||||
}
|
||||
}
|
||||
|
||||
func validChunkPlan(doc *SourceDocument) ChunkPlan {
|
||||
return ChunkPlan{
|
||||
SourceDigest: doc.Digest,
|
||||
|
||||
@@ -54,54 +54,62 @@ func (i DocumentIndex) ValidateRef(ref SourceRef) error {
|
||||
}
|
||||
|
||||
func ValidateDocument(doc *SourceDocument) error {
|
||||
_, err := validateDocument(doc)
|
||||
return err
|
||||
}
|
||||
|
||||
// validateDocument builds the position index while checking unit identities,
|
||||
// then uses it for all self-references and callers' bulk range operations.
|
||||
func validateDocument(doc *SourceDocument) (DocumentIndex, error) {
|
||||
if doc == nil {
|
||||
return fmt.Errorf("source document must not be nil")
|
||||
return DocumentIndex{}, fmt.Errorf("source document must not be nil")
|
||||
}
|
||||
if isBlank(doc.ID) {
|
||||
return fmt.Errorf("source document id must not be empty")
|
||||
return DocumentIndex{}, fmt.Errorf("source document id must not be empty")
|
||||
}
|
||||
if hasSurroundingWhitespace(doc.ID) {
|
||||
return fmt.Errorf("source document id %q must not contain leading or trailing whitespace", doc.ID)
|
||||
return DocumentIndex{}, fmt.Errorf("source document id %q must not contain leading or trailing whitespace", doc.ID)
|
||||
}
|
||||
if isBlank(doc.Kind) {
|
||||
return fmt.Errorf("source document kind must not be empty")
|
||||
return DocumentIndex{}, fmt.Errorf("source document kind must not be empty")
|
||||
}
|
||||
if isBlank(doc.Format) {
|
||||
return fmt.Errorf("source document format must not be empty")
|
||||
return DocumentIndex{}, fmt.Errorf("source document format must not be empty")
|
||||
}
|
||||
if isBlank(doc.Digest) {
|
||||
return fmt.Errorf("source document digest must not be empty")
|
||||
return DocumentIndex{}, fmt.Errorf("source document digest must not be empty")
|
||||
}
|
||||
if len(doc.Units) == 0 {
|
||||
return fmt.Errorf("source document units must not be empty")
|
||||
return DocumentIndex{}, fmt.Errorf("source document units must not be empty")
|
||||
}
|
||||
|
||||
seenUnitIDs := make(map[int]struct{}, len(doc.Units))
|
||||
positions := make(map[int]int, len(doc.Units))
|
||||
for i, unit := range doc.Units {
|
||||
if unit.ID <= 0 {
|
||||
return fmt.Errorf("source unit[%d].id must be positive", i)
|
||||
return DocumentIndex{}, fmt.Errorf("source unit[%d].id must be positive", i)
|
||||
}
|
||||
if isBlank(unit.Kind) {
|
||||
return fmt.Errorf("source unit[%d].kind must not be empty", i)
|
||||
return DocumentIndex{}, fmt.Errorf("source unit[%d].kind must not be empty", i)
|
||||
}
|
||||
if isBlank(unit.Text) {
|
||||
return fmt.Errorf("source unit[%d].text must not be empty", i)
|
||||
return DocumentIndex{}, fmt.Errorf("source unit[%d].text must not be empty", i)
|
||||
}
|
||||
if _, ok := seenUnitIDs[unit.ID]; ok {
|
||||
return fmt.Errorf("source unit id %d is duplicated", unit.ID)
|
||||
if _, ok := positions[unit.ID]; ok {
|
||||
return DocumentIndex{}, fmt.Errorf("source unit id %d is duplicated", unit.ID)
|
||||
}
|
||||
seenUnitIDs[unit.ID] = struct{}{}
|
||||
positions[unit.ID] = i
|
||||
}
|
||||
index := DocumentIndex{documentID: doc.ID, positions: positions, hasDocument: true}
|
||||
for i, unit := range doc.Units {
|
||||
if err := ValidateRef(doc, unit.Ref); err != nil {
|
||||
return fmt.Errorf("source unit[%d].ref: %w", i, err)
|
||||
if err := index.ValidateRef(unit.Ref); err != nil {
|
||||
return DocumentIndex{}, fmt.Errorf("source unit[%d].ref: %w", i, err)
|
||||
}
|
||||
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
|
||||
return fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID)
|
||||
return DocumentIndex{}, fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func ValidateRef(doc *SourceDocument, ref SourceRef) error {
|
||||
|
||||
@@ -11,9 +11,6 @@ func validateAndMaterializeChunkPlan(doc *source.SourceDocument, plan source.Chu
|
||||
if err != nil {
|
||||
return source.ChunkPlan{}, nil, fmt.Errorf("canonicalize chunk plan: %w", err)
|
||||
}
|
||||
if err := source.ValidateChunkPlan(doc, canonical); err != nil {
|
||||
return source.ChunkPlan{}, nil, err
|
||||
}
|
||||
chunks, err := source.MaterializeChunkPlan(doc, canonical)
|
||||
if err != nil {
|
||||
return source.ChunkPlan{}, nil, err
|
||||
|
||||
Reference in New Issue
Block a user