Produce the requested section count when feasible

This commit is contained in:
2026-11-16 04:02:22 +00:00
parent 27f494d4e4
commit 43ce620122
6 changed files with 188 additions and 104 deletions

View File

@@ -1,6 +1,9 @@
package chunking
import (
"errors"
"math/bits"
"strconv"
"strings"
"testing"
@@ -301,8 +304,9 @@ func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
var infeasible *InfeasibleTargetError
if !errors.As(err, &infeasible) || infeasible.SegmentCount != len(segments) {
t.Fatalf("expected typed segment-count error, got %v", err)
}
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
t.Fatalf("unexpected error: %v", err)
@@ -318,14 +322,94 @@ func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
)
_, err := chunker.ChunkTranscript(makeTranscript(segments))
if err == nil {
t.Fatal("expected error for impossible target_sections")
var infeasible *InfeasibleTargetError
if !errors.As(err, &infeasible) || infeasible.MinimumSections != 3 {
t.Fatalf("expected typed minimum-section error, got %v", err)
}
if !strings.Contains(err.Error(), "need at least 3 sections") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestChunkExplicitTargetKeepsFeasibleSuffix(t *testing.T) {
segments := makeSegments([]string{"first", "second", "third"})
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: 6, MinSectionTokens: 4, TargetSections: intPtr(2)},
&mapTokenEstimator{byText: map[string]int{"first": 2, "second": 5, "third": 1}},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if err != nil || len(sections) != 2 || sections[0].EstimatedTokens != 2 || sections[1].EstimatedTokens != 6 {
t.Fatalf("wanted weights [2, 6], got %+v (%v)", sections, err)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
}
func TestChunkExplicitTargetsMatchEnumeratedPartitions(t *testing.T) {
// Enumerating cut masks gives an independent oracle for feasible counts,
// including zero-weight segments around oversized singletons.
const size, maximum = 5, 2
texts := make([]string, size)
for i := range texts {
texts[i] = strconv.Itoa(i)
}
segments := makeSegments(texts)
for sample := range 1 << (2 * size) {
var weights [size]int
byText := make(map[string]int, size)
for i, text := range texts {
weights[i] = (sample >> (2 * i)) & 3
byText[text] = weights[i]
}
var feasible [size + 1]bool
for cuts := range 1 << (size - 1) {
start, total := 0, 0
valid := true
for i, weight := range weights {
total += weight
if i == size-1 || cuts&(1<<i) != 0 {
if i > start && total > maximum {
valid = false
break
}
start, total = i+1, 0
}
}
if valid {
feasible[1+bits.OnesCount(uint(cuts))] = true
}
}
for target := 1; target <= size; target++ {
chunker := NewChunkerWithEstimator(
ChunkingConfig{MaxSectionTokens: maximum, TargetSections: &target},
&mapTokenEstimator{byText: byText},
)
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
if !feasible[target] {
var infeasible *InfeasibleTargetError
if !errors.As(err, &infeasible) {
t.Fatalf("weights %v target %d should be infeasible, got %+v (%v)", weights, target, sections, err)
}
continue
}
if err != nil || len(sections) != target {
t.Fatalf("weights %v target %d should be feasible, got %+v (%v)", weights, target, sections, err)
}
assertSegmentCoverageAndOrder(t, segments, sections)
assertSectionMetadataConsistent(t, sections)
for _, section := range sections {
total := 0
for _, segment := range section.Segments {
total += weights[segment.ID-1]
}
if total != section.EstimatedTokens || (total > maximum && len(section.Segments) != 1) {
t.Fatalf("weights %v target %d yielded invalid section %+v", weights, target, section)
}
}
}
}
}
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
segments := makeSegments(texts)