diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 0dcbc548..68f1ab7c 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -319,6 +319,11 @@ run contracts, pipeline, CLI, and module integration tests after removal. ## Finding 6: Use Indexed Source Validation In The Chunk-Plan Hot Path +**Status: Complete.** Document validation builds one unit-position map, and plan +materialization reuses checked endpoints without rescanning or duplicate +framework validation. Source, chunk-map, and pipeline tests cover retained +ordering, ownership, malformed-input behavior, and nonmonotonic IDs. + Priority: P2. Relevant stages: 16, 24, 28–29, and 37. ### Evidence And Complexity diff --git a/internal/core/source/chunk_plan.go b/internal/core/source/chunk_plan.go index ac4a6c98..79f1263e 100644 --- a/internal/core/source/chunk_plan.go +++ b/internal/core/source/chunk_plan.go @@ -117,51 +117,63 @@ func CanonicalizeChunkPlan(plan ChunkPlan) (ChunkPlan, error) { // ValidateChunkPlan validates a canonical plan against the current source. // Ranges may contain gaps or overlap, but their start positions must increase. func ValidateChunkPlan(doc *SourceDocument, plan ChunkPlan) error { - if err := ValidateDocument(doc); err != nil { - return fmt.Errorf("source document: %w", err) + _, err := validateChunkPlan(doc, plan) + return err +} + +type chunkPositions struct{ start, end int } + +// Retain resolved positions so materialization does not rescan the document or +// repeat validation. Ordering is by document position, never numeric unit ID. +func validateChunkPlan(doc *SourceDocument, plan ChunkPlan) ([]chunkPositions, error) { + index, err := validateDocument(doc) + if err != nil { + return nil, fmt.Errorf("source document: %w", err) } if plan.SourceDigest != doc.Digest { - return fmt.Errorf("chunk plan source_digest %q does not match source document digest %q", plan.SourceDigest, doc.Digest) + return nil, fmt.Errorf("chunk plan source_digest %q does not match source document digest %q", plan.SourceDigest, doc.Digest) } if len(plan.Ranges) == 0 { - return fmt.Errorf("chunk plan ranges must not be empty") + return nil, fmt.Errorf("chunk plan ranges must not be empty") } if err := ValidateChunkAnnotations(plan.Annotations); err != nil { - return fmt.Errorf("chunk plan annotations: %w", err) + return nil, fmt.Errorf("chunk plan annotations: %w", err) } + positions := make([]chunkPositions, len(plan.Ranges)) previousStart := -1 for i, chunkRange := range plan.Ranges { - start, ok := UnitIndex(doc, chunkRange.StartUnitID) + start, ok := index.Position(chunkRange.StartUnitID) if !ok { - return fmt.Errorf("chunk plan range[%d] start_unit_id %d was not found", i, chunkRange.StartUnitID) + return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d was not found", i, chunkRange.StartUnitID) } - end, ok := UnitIndex(doc, chunkRange.EndUnitID) + end, ok := index.Position(chunkRange.EndUnitID) if !ok { - return fmt.Errorf("chunk plan range[%d] end_unit_id %d was not found", i, chunkRange.EndUnitID) + return nil, fmt.Errorf("chunk plan range[%d] end_unit_id %d was not found", i, chunkRange.EndUnitID) } if start > end { - return fmt.Errorf("chunk plan range[%d] start_unit_id %d appears after end_unit_id %d", i, chunkRange.StartUnitID, chunkRange.EndUnitID) + return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d appears after end_unit_id %d", i, chunkRange.StartUnitID, chunkRange.EndUnitID) } if start <= previousStart { - return fmt.Errorf("chunk plan range[%d] start_unit_id %d does not appear after the previous range start", i, chunkRange.StartUnitID) + return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d does not appear after the previous range start", i, chunkRange.StartUnitID) } if err := ValidateChunkAnnotations(chunkRange.Annotations); err != nil { - return fmt.Errorf("chunk plan range[%d] annotations: %w", i, err) + return nil, fmt.Errorf("chunk plan range[%d] annotations: %w", i, err) } + positions[i] = chunkPositions{start: start, end: end} previousStart = start } - return nil + return positions, nil } // MaterializeChunkPlan deterministically expands a validated plan into chunks. func MaterializeChunkPlan(doc *SourceDocument, plan ChunkPlan) ([]Chunk, error) { - if err := ValidateChunkPlan(doc, plan); err != nil { + positions, err := validateChunkPlan(doc, plan) + if err != nil { return nil, err } chunks := make([]Chunk, 0, len(plan.Ranges)) for index, chunkRange := range plan.Ranges { - start, _ := UnitIndex(doc, chunkRange.StartUnitID) - end, _ := UnitIndex(doc, chunkRange.EndUnitID) + start, end := positions[index].start, positions[index].end units, err := cloneSourceUnits(doc.Units[start : end+1]) if err != nil { return nil, fmt.Errorf("clone chunk plan range[%d] units: %w", index, err) diff --git a/internal/core/source/chunk_plan_test.go b/internal/core/source/chunk_plan_test.go index e6962399..33fdadc2 100644 --- a/internal/core/source/chunk_plan_test.go +++ b/internal/core/source/chunk_plan_test.go @@ -312,6 +312,33 @@ func planDocument() *SourceDocument { return doc } +func TestChunkPlanUsesDocumentOrderForNonmonotonicUnitIDs(t *testing.T) { + doc := planDocument() + doc.Units = doc.Units[:3] + for i, id := range []int{90, 7, 42} { + doc.Units[i].ID = id + doc.Units[i].Ref = SourceRef{SourceID: doc.ID, StartUnitID: id, EndUnitID: id} + } + plan := ChunkPlan{SourceDigest: doc.Digest, Ranges: []ChunkRange{{StartUnitID: 90, EndUnitID: 42}, {StartUnitID: 7, EndUnitID: 42}}} + if err := ValidateChunkPlan(doc, plan); err != nil { + t.Fatal(err) + } + chunks, err := MaterializeChunkPlan(doc, plan) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{90, 7, 42}) || !reflect.DeepEqual(unitIDs(chunks[1].Units), []int{7, 42}) { + t.Fatalf("nonmonotonic IDs changed materialization order: %#v", chunks) + } + plan.Ranges = []ChunkRange{{StartUnitID: 42, EndUnitID: 90}} + if err := ValidateChunkPlan(doc, plan); err == nil { + t.Fatal("reversed document positions passed validation") + } + if _, err := MaterializeChunkPlan(doc, plan); err == nil { + t.Fatal("reversed document positions materialized") + } +} + func validChunkPlan(doc *SourceDocument) ChunkPlan { return ChunkPlan{ SourceDigest: doc.Digest, diff --git a/internal/core/source/validation.go b/internal/core/source/validation.go index 234d5e87..09ab46bd 100644 --- a/internal/core/source/validation.go +++ b/internal/core/source/validation.go @@ -54,54 +54,62 @@ func (i DocumentIndex) ValidateRef(ref SourceRef) error { } func ValidateDocument(doc *SourceDocument) error { + _, err := validateDocument(doc) + return err +} + +// validateDocument builds the position index while checking unit identities, +// then uses it for all self-references and callers' bulk range operations. +func validateDocument(doc *SourceDocument) (DocumentIndex, error) { if doc == nil { - return fmt.Errorf("source document must not be nil") + return DocumentIndex{}, fmt.Errorf("source document must not be nil") } if isBlank(doc.ID) { - return fmt.Errorf("source document id must not be empty") + return DocumentIndex{}, fmt.Errorf("source document id must not be empty") } if hasSurroundingWhitespace(doc.ID) { - return fmt.Errorf("source document id %q must not contain leading or trailing whitespace", doc.ID) + return DocumentIndex{}, fmt.Errorf("source document id %q must not contain leading or trailing whitespace", doc.ID) } if isBlank(doc.Kind) { - return fmt.Errorf("source document kind must not be empty") + return DocumentIndex{}, fmt.Errorf("source document kind must not be empty") } if isBlank(doc.Format) { - return fmt.Errorf("source document format must not be empty") + return DocumentIndex{}, fmt.Errorf("source document format must not be empty") } if isBlank(doc.Digest) { - return fmt.Errorf("source document digest must not be empty") + return DocumentIndex{}, fmt.Errorf("source document digest must not be empty") } if len(doc.Units) == 0 { - return fmt.Errorf("source document units must not be empty") + return DocumentIndex{}, fmt.Errorf("source document units must not be empty") } - seenUnitIDs := make(map[int]struct{}, len(doc.Units)) + positions := make(map[int]int, len(doc.Units)) for i, unit := range doc.Units { if unit.ID <= 0 { - return fmt.Errorf("source unit[%d].id must be positive", i) + return DocumentIndex{}, fmt.Errorf("source unit[%d].id must be positive", i) } if isBlank(unit.Kind) { - return fmt.Errorf("source unit[%d].kind must not be empty", i) + return DocumentIndex{}, fmt.Errorf("source unit[%d].kind must not be empty", i) } if isBlank(unit.Text) { - return fmt.Errorf("source unit[%d].text must not be empty", i) + return DocumentIndex{}, fmt.Errorf("source unit[%d].text must not be empty", i) } - if _, ok := seenUnitIDs[unit.ID]; ok { - return fmt.Errorf("source unit id %d is duplicated", unit.ID) + if _, ok := positions[unit.ID]; ok { + return DocumentIndex{}, fmt.Errorf("source unit id %d is duplicated", unit.ID) } - seenUnitIDs[unit.ID] = struct{}{} + positions[unit.ID] = i } + index := DocumentIndex{documentID: doc.ID, positions: positions, hasDocument: true} for i, unit := range doc.Units { - if err := ValidateRef(doc, unit.Ref); err != nil { - return fmt.Errorf("source unit[%d].ref: %w", i, err) + if err := index.ValidateRef(unit.Ref); err != nil { + return DocumentIndex{}, fmt.Errorf("source unit[%d].ref: %w", i, err) } if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID { - return fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID) + return DocumentIndex{}, fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID) } } - return nil + return index, nil } func ValidateRef(doc *SourceDocument, ref SourceRef) error { diff --git a/internal/framework/pipeline/chunk_validation.go b/internal/framework/pipeline/chunk_validation.go index 3b9b10f8..66eb718e 100644 --- a/internal/framework/pipeline/chunk_validation.go +++ b/internal/framework/pipeline/chunk_validation.go @@ -11,9 +11,6 @@ func validateAndMaterializeChunkPlan(doc *source.SourceDocument, plan source.Chu if err != nil { return source.ChunkPlan{}, nil, fmt.Errorf("canonicalize chunk plan: %w", err) } - if err := source.ValidateChunkPlan(doc, canonical); err != nil { - return source.ChunkPlan{}, nil, err - } chunks, err := source.MaterializeChunkPlan(doc, canonical) if err != nil { return source.ChunkPlan{}, nil, err