Index source positions during chunk plan validation

This commit is contained in:
2026-11-22 09:40:42 +00:00
parent 7ee7b997f0
commit 2429f0ae6a
5 changed files with 86 additions and 37 deletions

View File

@@ -117,51 +117,63 @@ func CanonicalizeChunkPlan(plan ChunkPlan) (ChunkPlan, error) {
// ValidateChunkPlan validates a canonical plan against the current source.
// Ranges may contain gaps or overlap, but their start positions must increase.
func ValidateChunkPlan(doc *SourceDocument, plan ChunkPlan) error {
if err := ValidateDocument(doc); err != nil {
return fmt.Errorf("source document: %w", err)
_, err := validateChunkPlan(doc, plan)
return err
}
type chunkPositions struct{ start, end int }
// Retain resolved positions so materialization does not rescan the document or
// repeat validation. Ordering is by document position, never numeric unit ID.
func validateChunkPlan(doc *SourceDocument, plan ChunkPlan) ([]chunkPositions, error) {
index, err := validateDocument(doc)
if err != nil {
return nil, fmt.Errorf("source document: %w", err)
}
if plan.SourceDigest != doc.Digest {
return fmt.Errorf("chunk plan source_digest %q does not match source document digest %q", plan.SourceDigest, doc.Digest)
return nil, fmt.Errorf("chunk plan source_digest %q does not match source document digest %q", plan.SourceDigest, doc.Digest)
}
if len(plan.Ranges) == 0 {
return fmt.Errorf("chunk plan ranges must not be empty")
return nil, fmt.Errorf("chunk plan ranges must not be empty")
}
if err := ValidateChunkAnnotations(plan.Annotations); err != nil {
return fmt.Errorf("chunk plan annotations: %w", err)
return nil, fmt.Errorf("chunk plan annotations: %w", err)
}
positions := make([]chunkPositions, len(plan.Ranges))
previousStart := -1
for i, chunkRange := range plan.Ranges {
start, ok := UnitIndex(doc, chunkRange.StartUnitID)
start, ok := index.Position(chunkRange.StartUnitID)
if !ok {
return fmt.Errorf("chunk plan range[%d] start_unit_id %d was not found", i, chunkRange.StartUnitID)
return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d was not found", i, chunkRange.StartUnitID)
}
end, ok := UnitIndex(doc, chunkRange.EndUnitID)
end, ok := index.Position(chunkRange.EndUnitID)
if !ok {
return fmt.Errorf("chunk plan range[%d] end_unit_id %d was not found", i, chunkRange.EndUnitID)
return nil, fmt.Errorf("chunk plan range[%d] end_unit_id %d was not found", i, chunkRange.EndUnitID)
}
if start > end {
return fmt.Errorf("chunk plan range[%d] start_unit_id %d appears after end_unit_id %d", i, chunkRange.StartUnitID, chunkRange.EndUnitID)
return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d appears after end_unit_id %d", i, chunkRange.StartUnitID, chunkRange.EndUnitID)
}
if start <= previousStart {
return fmt.Errorf("chunk plan range[%d] start_unit_id %d does not appear after the previous range start", i, chunkRange.StartUnitID)
return nil, fmt.Errorf("chunk plan range[%d] start_unit_id %d does not appear after the previous range start", i, chunkRange.StartUnitID)
}
if err := ValidateChunkAnnotations(chunkRange.Annotations); err != nil {
return fmt.Errorf("chunk plan range[%d] annotations: %w", i, err)
return nil, fmt.Errorf("chunk plan range[%d] annotations: %w", i, err)
}
positions[i] = chunkPositions{start: start, end: end}
previousStart = start
}
return nil
return positions, nil
}
// MaterializeChunkPlan deterministically expands a validated plan into chunks.
func MaterializeChunkPlan(doc *SourceDocument, plan ChunkPlan) ([]Chunk, error) {
if err := ValidateChunkPlan(doc, plan); err != nil {
positions, err := validateChunkPlan(doc, plan)
if err != nil {
return nil, err
}
chunks := make([]Chunk, 0, len(plan.Ranges))
for index, chunkRange := range plan.Ranges {
start, _ := UnitIndex(doc, chunkRange.StartUnitID)
end, _ := UnitIndex(doc, chunkRange.EndUnitID)
start, end := positions[index].start, positions[index].end
units, err := cloneSourceUnits(doc.Units[start : end+1])
if err != nil {
return nil, fmt.Errorf("clone chunk plan range[%d] units: %w", index, err)

View File

@@ -312,6 +312,33 @@ func planDocument() *SourceDocument {
return doc
}
func TestChunkPlanUsesDocumentOrderForNonmonotonicUnitIDs(t *testing.T) {
doc := planDocument()
doc.Units = doc.Units[:3]
for i, id := range []int{90, 7, 42} {
doc.Units[i].ID = id
doc.Units[i].Ref = SourceRef{SourceID: doc.ID, StartUnitID: id, EndUnitID: id}
}
plan := ChunkPlan{SourceDigest: doc.Digest, Ranges: []ChunkRange{{StartUnitID: 90, EndUnitID: 42}, {StartUnitID: 7, EndUnitID: 42}}}
if err := ValidateChunkPlan(doc, plan); err != nil {
t.Fatal(err)
}
chunks, err := MaterializeChunkPlan(doc, plan)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(unitIDs(chunks[0].Units), []int{90, 7, 42}) || !reflect.DeepEqual(unitIDs(chunks[1].Units), []int{7, 42}) {
t.Fatalf("nonmonotonic IDs changed materialization order: %#v", chunks)
}
plan.Ranges = []ChunkRange{{StartUnitID: 42, EndUnitID: 90}}
if err := ValidateChunkPlan(doc, plan); err == nil {
t.Fatal("reversed document positions passed validation")
}
if _, err := MaterializeChunkPlan(doc, plan); err == nil {
t.Fatal("reversed document positions materialized")
}
}
func validChunkPlan(doc *SourceDocument) ChunkPlan {
return ChunkPlan{
SourceDigest: doc.Digest,

View File

@@ -54,54 +54,62 @@ func (i DocumentIndex) ValidateRef(ref SourceRef) error {
}
func ValidateDocument(doc *SourceDocument) error {
_, err := validateDocument(doc)
return err
}
// validateDocument builds the position index while checking unit identities,
// then uses it for all self-references and callers' bulk range operations.
func validateDocument(doc *SourceDocument) (DocumentIndex, error) {
if doc == nil {
return fmt.Errorf("source document must not be nil")
return DocumentIndex{}, fmt.Errorf("source document must not be nil")
}
if isBlank(doc.ID) {
return fmt.Errorf("source document id must not be empty")
return DocumentIndex{}, fmt.Errorf("source document id must not be empty")
}
if hasSurroundingWhitespace(doc.ID) {
return fmt.Errorf("source document id %q must not contain leading or trailing whitespace", doc.ID)
return DocumentIndex{}, fmt.Errorf("source document id %q must not contain leading or trailing whitespace", doc.ID)
}
if isBlank(doc.Kind) {
return fmt.Errorf("source document kind must not be empty")
return DocumentIndex{}, fmt.Errorf("source document kind must not be empty")
}
if isBlank(doc.Format) {
return fmt.Errorf("source document format must not be empty")
return DocumentIndex{}, fmt.Errorf("source document format must not be empty")
}
if isBlank(doc.Digest) {
return fmt.Errorf("source document digest must not be empty")
return DocumentIndex{}, fmt.Errorf("source document digest must not be empty")
}
if len(doc.Units) == 0 {
return fmt.Errorf("source document units must not be empty")
return DocumentIndex{}, fmt.Errorf("source document units must not be empty")
}
seenUnitIDs := make(map[int]struct{}, len(doc.Units))
positions := make(map[int]int, len(doc.Units))
for i, unit := range doc.Units {
if unit.ID <= 0 {
return fmt.Errorf("source unit[%d].id must be positive", i)
return DocumentIndex{}, fmt.Errorf("source unit[%d].id must be positive", i)
}
if isBlank(unit.Kind) {
return fmt.Errorf("source unit[%d].kind must not be empty", i)
return DocumentIndex{}, fmt.Errorf("source unit[%d].kind must not be empty", i)
}
if isBlank(unit.Text) {
return fmt.Errorf("source unit[%d].text must not be empty", i)
return DocumentIndex{}, fmt.Errorf("source unit[%d].text must not be empty", i)
}
if _, ok := seenUnitIDs[unit.ID]; ok {
return fmt.Errorf("source unit id %d is duplicated", unit.ID)
if _, ok := positions[unit.ID]; ok {
return DocumentIndex{}, fmt.Errorf("source unit id %d is duplicated", unit.ID)
}
seenUnitIDs[unit.ID] = struct{}{}
positions[unit.ID] = i
}
index := DocumentIndex{documentID: doc.ID, positions: positions, hasDocument: true}
for i, unit := range doc.Units {
if err := ValidateRef(doc, unit.Ref); err != nil {
return fmt.Errorf("source unit[%d].ref: %w", i, err)
if err := index.ValidateRef(unit.Ref); err != nil {
return DocumentIndex{}, fmt.Errorf("source unit[%d].ref: %w", i, err)
}
if unit.Ref.StartUnitID != unit.ID || unit.Ref.EndUnitID != unit.ID {
return fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID)
return DocumentIndex{}, fmt.Errorf("source unit[%d].ref must identify source unit id %d", i, unit.ID)
}
}
return nil
return index, nil
}
func ValidateRef(doc *SourceDocument, ref SourceRef) error {

View File

@@ -11,9 +11,6 @@ func validateAndMaterializeChunkPlan(doc *source.SourceDocument, plan source.Chu
if err != nil {
return source.ChunkPlan{}, nil, fmt.Errorf("canonicalize chunk plan: %w", err)
}
if err := source.ValidateChunkPlan(doc, canonical); err != nil {
return source.ChunkPlan{}, nil, err
}
chunks, err := source.MaterializeChunkPlan(doc, canonical)
if err != nil {
return source.ChunkPlan{}, nil, err