Produce the requested section count when feasible
This commit is contained in:
@@ -1,6 +1,9 @@
|
||||
package chunking
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math/bits"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -301,8 +304,9 @@ func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
|
||||
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
var infeasible *InfeasibleTargetError
|
||||
if !errors.As(err, &infeasible) || infeasible.SegmentCount != len(segments) {
|
||||
t.Fatalf("expected typed segment-count error, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -318,14 +322,94 @@ func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
|
||||
)
|
||||
|
||||
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err == nil {
|
||||
t.Fatal("expected error for impossible target_sections")
|
||||
var infeasible *InfeasibleTargetError
|
||||
if !errors.As(err, &infeasible) || infeasible.MinimumSections != 3 {
|
||||
t.Fatalf("expected typed minimum-section error, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "need at least 3 sections") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkExplicitTargetKeepsFeasibleSuffix(t *testing.T) {
|
||||
segments := makeSegments([]string{"first", "second", "third"})
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: 6, MinSectionTokens: 4, TargetSections: intPtr(2)},
|
||||
&mapTokenEstimator{byText: map[string]int{"first": 2, "second": 5, "third": 1}},
|
||||
)
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if err != nil || len(sections) != 2 || sections[0].EstimatedTokens != 2 || sections[1].EstimatedTokens != 6 {
|
||||
t.Fatalf("wanted weights [2, 6], got %+v (%v)", sections, err)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
}
|
||||
|
||||
func TestChunkExplicitTargetsMatchEnumeratedPartitions(t *testing.T) {
|
||||
// Enumerating cut masks gives an independent oracle for feasible counts,
|
||||
// including zero-weight segments around oversized singletons.
|
||||
const size, maximum = 5, 2
|
||||
texts := make([]string, size)
|
||||
for i := range texts {
|
||||
texts[i] = strconv.Itoa(i)
|
||||
}
|
||||
segments := makeSegments(texts)
|
||||
for sample := range 1 << (2 * size) {
|
||||
var weights [size]int
|
||||
byText := make(map[string]int, size)
|
||||
for i, text := range texts {
|
||||
weights[i] = (sample >> (2 * i)) & 3
|
||||
byText[text] = weights[i]
|
||||
}
|
||||
var feasible [size + 1]bool
|
||||
for cuts := range 1 << (size - 1) {
|
||||
start, total := 0, 0
|
||||
valid := true
|
||||
for i, weight := range weights {
|
||||
total += weight
|
||||
if i == size-1 || cuts&(1<<i) != 0 {
|
||||
if i > start && total > maximum {
|
||||
valid = false
|
||||
break
|
||||
}
|
||||
start, total = i+1, 0
|
||||
}
|
||||
}
|
||||
if valid {
|
||||
feasible[1+bits.OnesCount(uint(cuts))] = true
|
||||
}
|
||||
}
|
||||
for target := 1; target <= size; target++ {
|
||||
chunker := NewChunkerWithEstimator(
|
||||
ChunkingConfig{MaxSectionTokens: maximum, TargetSections: &target},
|
||||
&mapTokenEstimator{byText: byText},
|
||||
)
|
||||
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
||||
if !feasible[target] {
|
||||
var infeasible *InfeasibleTargetError
|
||||
if !errors.As(err, &infeasible) {
|
||||
t.Fatalf("weights %v target %d should be infeasible, got %+v (%v)", weights, target, sections, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || len(sections) != target {
|
||||
t.Fatalf("weights %v target %d should be feasible, got %+v (%v)", weights, target, sections, err)
|
||||
}
|
||||
assertSegmentCoverageAndOrder(t, segments, sections)
|
||||
assertSectionMetadataConsistent(t, sections)
|
||||
for _, section := range sections {
|
||||
total := 0
|
||||
for _, segment := range section.Segments {
|
||||
total += weights[segment.ID-1]
|
||||
}
|
||||
if total != section.EstimatedTokens || (total > maximum && len(section.Segments) != 1) {
|
||||
t.Fatalf("weights %v target %d yielded invalid section %+v", weights, target, section)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
|
||||
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
|
||||
segments := makeSegments(texts)
|
||||
|
||||
Reference in New Issue
Block a user