Produce the requested section count when feasible

This commit is contained in:
2026-11-16 04:02:22 +00:00
parent 27f494d4e4
commit 43ce620122
6 changed files with 188 additions and 104 deletions

View File

@@ -2,6 +2,7 @@ package chunking
import (
"fmt"
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
@@ -100,66 +101,35 @@ func (c *Chunker) ChunkTranscript(transcript *schema.Transcript) ([]Section, err
totalTokens += segmentTokens[i]
}
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
var desiredSections int
useExplicitTarget := false
if c.config.TargetSections != nil {
desiredSections = *c.config.TargetSections
useExplicitTarget = true
if desiredSections <= 0 {
return nil, fmt.Errorf("target_sections must be positive, got %d", desiredSections)
}
if err := c.validateTargetSections(desiredSections, segmentTokens); err != nil {
return nil, err
}
} else {
desiredSections = ceilDiv(totalTokens, c.config.MaxSectionTokens)
if desiredSections < minPossibleSections {
desiredSections = minPossibleSections
}
if desiredSections < 1 {
desiredSections = 1
}
if desiredSections > len(transcript.Segments) {
desiredSections = len(transcript.Segments)
target := *c.config.TargetSections
if target <= 0 {
return nil, fmt.Errorf("target_sections must be positive, got %d", target)
}
return c.buildSectionsWithExplicitTarget(transcript.Segments, segmentTokens, target, ceilDiv(totalTokens, target))
}
targetTokensPerSection := ceilDiv(totalTokens, desiredSections)
if useExplicitTarget {
return c.buildSectionsWithExplicitTarget(
transcript.Segments,
segmentTokens,
desiredSections,
targetTokensPerSection,
)
}
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, targetTokensPerSection), nil
desiredSections := ceilDiv(totalTokens, c.config.MaxSectionTokens)
desiredSections = max(desiredSections, c.calculateMinPossibleSections(segmentTokens), 1)
desiredSections = min(desiredSections, len(transcript.Segments))
return c.buildSectionsBalanced(transcript.Segments, segmentTokens, ceilDiv(totalTokens, desiredSections)), nil
}
// validateTargetSections checks if the target section count is achievable.
func (c *Chunker) validateTargetSections(target int, segmentTokens []int) error {
// Maximum possible sections: limited by segment count
maxPossibleSections := len(segmentTokens)
// InfeasibleTargetError indicates that a requested count cannot partition the
// normalized segments into nonempty, token-bounded sections.
type InfeasibleTargetError struct {
Target int
SegmentCount int
MinimumSections int
MaxSectionTokens int
}
if target > maxPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: cannot have more sections than segments (%d)",
target, maxPossibleSections)
func (e *InfeasibleTargetError) Error() string {
if e.Target > e.SegmentCount {
return fmt.Sprintf("target_sections (%d) is impossible: cannot have more sections than segments (%d)", e.Target, e.SegmentCount)
}
// Minimum possible sections: each segment must fit within max bounds
minPossibleSections := c.calculateMinPossibleSections(segmentTokens)
if target < minPossibleSections {
return fmt.Errorf(
"target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
target, minPossibleSections, c.config.MaxSectionTokens)
}
return nil
return fmt.Sprintf("target_sections (%d) is impossible: need at least %d sections to respect max_section_tokens (%d)",
e.Target, e.MinimumSections, e.MaxSectionTokens)
}
// buildSectionsBalanced creates sections with a deterministic single-pass policy.
@@ -245,59 +215,63 @@ func (c *Chunker) calculateMinPossibleSections(segmentTokens []int) int {
return sections
}
// buildSectionsWithExplicitTarget builds exactly desiredSections when feasible.
// farthestLegalEnds records the exclusive end of the longest legal section
// starting at each segment, with oversized segments admitted only alone.
func farthestLegalEnds(segmentTokens []int, maximum int) []int {
ends := make([]int, len(segmentTokens))
end, windowTokens := 0, 0
for start, weight := range segmentTokens {
if weight > maximum {
end = start + 1
ends[start] = end
continue
}
for end < len(segmentTokens) && segmentTokens[end] <= maximum-windowTokens {
windowTokens += segmentTokens[end]
end++
}
ends[start] = end
windowTokens -= weight
}
return ends
}
// buildSectionsWithExplicitTarget builds exactly target nonempty sections when feasible.
func (c *Chunker) buildSectionsWithExplicitTarget(
segments []schema.Segment,
segmentTokens []int,
desiredSections int,
targetTokensPerSection int,
target, targetTokensPerSection int,
) ([]Section, error) {
n := len(segments)
cursor := 0
sections := make([]Section, 0, desiredSections)
for sectionIdx := 0; sectionIdx < desiredSections; sectionIdx++ {
if cursor >= n {
break
}
remainingSectionsAfter := desiredSections - sectionIdx - 1
currentSegments := []schema.Segment{segments[cursor]}
currentTokens := segmentTokens[cursor]
cursor++
for cursor < n {
remainingSegments := n - cursor
// Reserve one segment per future section to avoid empty sections.
if remainingSegments == remainingSectionsAfter {
break
}
nextTokens := segmentTokens[cursor]
if currentTokens+nextTokens > c.config.MaxSectionTokens {
break
}
if targetTokensPerSection == 0 || currentTokens < targetTokensPerSection {
currentSegments = append(currentSegments, segments[cursor])
currentTokens += nextTokens
cursor++
continue
}
break
}
sections = append(sections, c.buildSection(len(sections), currentSegments, currentTokens))
if target > n {
return nil, &InfeasibleTargetError{Target: target, SegmentCount: n}
}
farthest := farthestLegalEnds(segmentTokens, c.config.MaxSectionTokens)
minimum := make([]int, n+1)
for start := n - 1; start >= 0; start-- {
minimum[start] = 1 + minimum[farthest[start]]
}
if target < minimum[0] {
return nil, &InfeasibleTargetError{Target: target, SegmentCount: n, MinimumSections: minimum[0], MaxSectionTokens: c.config.MaxSectionTokens}
}
if cursor != n || len(sections) != desiredSections {
return nil, fmt.Errorf(
"target_sections (%d) is impossible under current constraints (got %d sections)",
desiredSections,
len(sections),
)
sections := make([]Section, 0, target)
start := 0
for i := range target {
remaining := target - i - 1
limit := min(farthest[start], n-remaining)
total := 0
for end := start + 1; end <= limit; end++ {
total += segmentTokens[end-1]
// Aim for the average where it leaves a feasible suffix. At
// the limit the suffix fits by construction.
if end == limit || (targetTokensPerSection > 0 && total >= targetTokensPerSection && minimum[end] <= remaining) {
sections = append(sections, c.buildSection(i, slices.Clone(segments[start:end]), total))
start = end
break
}
}
}
return sections, nil
}
@@ -308,5 +282,5 @@ func ceilDiv(numerator int, denominator int) int {
if numerator <= 0 {
return 0
}
return (numerator + denominator - 1) / denominator
return 1 + (numerator-1)/denominator
}

View File

@@ -1,6 +1,9 @@
package chunking
import (
"errors"
"math/bits"
"strconv"
"strings"
"testing"
@@ -301,8 +304,9 @@ func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
var infeasible *InfeasibleTargetError
if !errors.As(err, &infeasible) || infeasible.SegmentCount != len(segments) {
t.Fatalf("expected typed segment-count error, got %v", err)
}
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
t.Fatalf("unexpected error: %v", err)
@@ -318,14 +322,94 @@ func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
)
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
var infeasible *InfeasibleTargetError
if !errors.As(err, &infeasible) || infeasible.MinimumSections != 3 {
t.Fatalf("expected typed minimum-section error, got %v", err)
}
if !strings.Contains(err.Error(), "need at least 3 sections") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkExplicitTargetKeepsFeasibleSuffix(t *testing.T) {
segments := makeSegments([]string{"first", "second", "third"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 6, MinSectionTokens: 4, TargetSections: intPtr(2)},
&mapTokenEstimator{byText: map[string]int{"first": 2, "second": 5, "third": 1}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil || len(sections) != 2 || sections[0].EstimatedTokens != 2 || sections[1].EstimatedTokens != 6 {
t.Fatalf("wanted weights [2, 6], got %+v (%v)", sections, err)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkExplicitTargetsMatchEnumeratedPartitions(t *testing.T) {
// Enumerating cut masks gives an independent oracle for feasible counts,
// including zero-weight segments around oversized singletons.
const size, maximum = 5, 2
texts := make([]string, size)
for i := range texts {
texts[i] = strconv.Itoa(i)
}
segments := makeSegments(texts)
for sample := range 1 << (2 * size) {
var weights [size]int
byText := make(map[string]int, size)
for i, text := range texts {
weights[i] = (sample >> (2 * i)) & 3
byText[text] = weights[i]
}
var feasible [size + 1]bool
for cuts := range 1 << (size - 1) {
start, total := 0, 0
valid := true
for i, weight := range weights {
total += weight
if i == size-1 || cuts&(1<<i) != 0 {
if i > start && total > maximum {
valid = false
break
}
start, total = i+1, 0
}
}
if valid {
feasible[1+bits.OnesCount(uint(cuts))] = true
}
}
for target := 1; target <= size; target++ {
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: maximum, TargetSections: &target},
&mapTokenEstimator{byText: byText},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if !feasible[target] {
var infeasible *InfeasibleTargetError
if !errors.As(err, &infeasible) {
t.Fatalf("weights %v target %d should be infeasible, got %+v (%v)", weights, target, sections, err)
}
continue
}
if err != nil || len(sections) != target {
t.Fatalf("weights %v target %d should be feasible, got %+v (%v)", weights, target, sections, err)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
for _, section := range sections {
total := 0
for _, segment := range section.Segments {
total += weights[segment.ID-1]
}
if total != section.EstimatedTokens || (total > maximum && len(section.Segments) != 1) {
t.Fatalf("weights %v target %d yielded invalid section %+v", weights, target, section)
}
}
}
}
}
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
segments := makeSegments(texts)