Produce the requested section count when feasible
This commit is contained in:
@@ -2,6 +2,7 @@ package chunking
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
@@ -100,66 +101,35 @@ func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, err
|
||||
totalTokens += segmentTokens[i]
|
||||
}
|
||||
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
var desiredSections int
|
||||
useExplicitTarget := false
|
||||
if c.config.TargetSections != nil {
|
||||
desiredSections = *c.config.TargetSections
|
||||
useExplicitTarget = true
|
||||
if desiredSections <= 0 {
|
||||
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
|
||||
}
|
||||
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
|
||||
if desiredSections < minPossibleSections {
|
||||
desiredSections = minPossibleSections
|
||||
}
|
||||
if desiredSections < 1 {
|
||||
desiredSections = 1
|
||||
}
|
||||
if desiredSections > len(transcript.Segments) {
|
||||
desiredSections = len(transcript.Segments)
|
||||
target := *c.config.TargetSections
|
||||
if target <= 0 {
|
||||
return nil, fmt.Errorf("target_sections must be positive, got %d", target)
|
||||
}
|
||||
return c.buildSectionsWithExplicitTarget(transcript.Segments, segmentTokens, target, ceilDiv(totalTokens, target))
|
||||
}
|
||||
|
||||
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
|
||||
if useExplicitTarget {
|
||||
return c.buildSectionsWithExplicitTarget(
|
||||
transcript.Segments,
|
||||
segmentTokens,
|
||||
desiredSections,
|
||||
targetTokensPerSection,
|
||||
)
|
||||
}
|
||||
|
||||
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
|
||||
desiredSections := ceilDiv(totalTokens, c.config.MaxSectionTokens)
|
||||
desiredSections = max(desiredSections, c.calculateMinPossibleSections(segmentTokens), 1)
|
||||
desiredSections = min(desiredSections, len(transcript.Segments))
|
||||
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, ceilDiv(totalTokens, desiredSections)), nil
|
||||
}
|
||||
|
||||
// validateTargetSections checks if the target section count is achievable.
|
||||
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
|
||||
// Maximum possible sections: limited by segment count
|
||||
maxPossibleSections := len(segmentTokens)
|
||||
// InfeasibleTargetError indicates that a requested count cannot partition the
|
||||
// normalized segments into nonempty, token-bounded sections.
|
||||
type InfeasibleTargetError struct {
|
||||
Target int
|
||||
SegmentCount int
|
||||
MinimumSections int
|
||||
MaxSectionTokens int
|
||||
}
|
||||
|
||||
if target > maxPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
|
||||
target, maxPossibleSections)
|
||||
func (e *InfeasibleTargetError) Error() string {
|
||||
if e.Target > e.SegmentCount {
|
||||
return fmt.Sprintf("target_sections (%d) is impossible: cannot have more sections than segments (%d)", e.Target, e.SegmentCount)
|
||||
}
|
||||
|
||||
// Minimum possible sections: each segment must fit within max bounds
|
||||
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
|
||||
|
||||
if target < minPossibleSections {
|
||||
return fmt.Errorf(
|
||||
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
|
||||
target, minPossibleSections, c.config.MaxSectionTokens)
|
||||
}
|
||||
|
||||
return nil
|
||||
return fmt.Sprintf("target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
|
||||
e.Target, e.MinimumSections, e.MaxSectionTokens)
|
||||
}
|
||||
|
||||
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
|
||||
@@ -245,59 +215,63 @@ func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
|
||||
return sections
|
||||
}
|
||||
|
||||
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
|
||||
// farthestLegalEnds records the exclusive end of the longest legal section
|
||||
// starting at each segment, with oversized segments admitted only alone.
|
||||
func farthestLegalEnds(segmentTokens []int, maximum int) []int {
|
||||
ends := make([]int, len(segmentTokens))
|
||||
end, windowTokens := 0, 0
|
||||
for start, weight := range segmentTokens {
|
||||
if weight > maximum {
|
||||
end = start + 1
|
||||
ends[start] = end
|
||||
continue
|
||||
}
|
||||
for end < len(segmentTokens) && segmentTokens[end] <= maximum-windowTokens {
|
||||
windowTokens += segmentTokens[end]
|
||||
end++
|
||||
}
|
||||
ends[start] = end
|
||||
windowTokens -= weight
|
||||
}
|
||||
return ends
|
||||
}
|
||||
|
||||
// buildSectionsWithExplicitTarget builds exactly target nonempty sections when feasible.
|
||||
func (c *Chunker) buildSectionsWithExplicitTarget(
|
||||
segments []schema.Segment,
|
||||
segmentTokens []int,
|
||||
desiredSections int,
|
||||
targetTokensPerSection int,
|
||||
target, targetTokensPerSection int,
|
||||
) ([]Section, error) {
|
||||
n := len(segments)
|
||||
cursor := 0
|
||||
sections := make([]Section, 0, desiredSections)
|
||||
|
||||
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
|
||||
if cursor >= n {
|
||||
break
|
||||
}
|
||||
|
||||
remainingSectionsAfter := desiredSections - sectionIdx - 1
|
||||
currentSegments := []schema.Segment{segments[cursor]}
|
||||
currentTokens := segmentTokens[cursor]
|
||||
cursor++
|
||||
|
||||
for cursor < n {
|
||||
remainingSegments := n - cursor
|
||||
|
||||
// Reserve one segment per future section to avoid empty sections.
|
||||
if remainingSegments == remainingSectionsAfter {
|
||||
break
|
||||
}
|
||||
|
||||
nextTokens := segmentTokens[cursor]
|
||||
if currentTokens+nextTokens > c.config.MaxSectionTokens {
|
||||
break
|
||||
}
|
||||
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
|
||||
currentSegments = append(currentSegments, segments[cursor])
|
||||
currentTokens += nextTokens
|
||||
cursor++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
|
||||
if target > n {
|
||||
return nil, &InfeasibleTargetError{Target: target, SegmentCount: n}
|
||||
}
|
||||
farthest := farthestLegalEnds(segmentTokens, c.config.MaxSectionTokens)
|
||||
minimum := make([]int, n+1)
|
||||
for start := n - 1; start >= 0; start-- {
|
||||
minimum[start] = 1 + minimum[farthest[start]]
|
||||
}
|
||||
if target < minimum[0] {
|
||||
return nil, &InfeasibleTargetError{Target: target, SegmentCount: n, MinimumSections: minimum[0], MaxSectionTokens: c.config.MaxSectionTokens}
|
||||
}
|
||||
|
||||
if cursor != n || len(sections) != desiredSections {
|
||||
return nil, fmt.Errorf(
|
||||
"target_sections (%d) is impossible under current constraints (got %d sections)",
|
||||
desiredSections,
|
||||
len(sections),
|
||||
)
|
||||
sections := make([]Section, 0, target)
|
||||
start := 0
|
||||
for i := range target {
|
||||
remaining := target - i - 1
|
||||
limit := min(farthest[start], n-remaining)
|
||||
total := 0
|
||||
for end := start + 1; end <= limit; end++ {
|
||||
total += segmentTokens[end-1]
|
||||
// Aim for the average where it leaves a feasible suffix. At
|
||||
// the limit the suffix fits by construction.
|
||||
if end == limit || (targetTokensPerSection > 0 && total >= targetTokensPerSection && minimum[end] <= remaining) {
|
||||
sections = append(sections, c.buildSection(i, slices.Clone(segments[start:end]), total))
|
||||
start = end
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sections, nil
|
||||
}
|
||||
|
||||
@@ -308,5 +282,5 @@ func ceilDiv(numerator int, denominator int) int {
|
||||
if numerator <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (numerator + denominator - 1) / denominator
|
||||
return 1 + (numerator-1)/denominator
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user