Normalize nested intervals and build merges incrementally

This commit is contained in:
2026-11-16 03:59:44 +00:00
parent f465496dd8
commit 27f494d4e4
6 changed files with 173 additions and 229 deletions

View File

@@ -23,24 +23,39 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator {
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
words, punctuation := e.CountComponents(text)
return words + punctuation/2
var count TokenCount
count.Add(text)
return count.Estimate()
}
// CountComponents returns unrounded counts for budgets composed from fragments.
func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
inWord := false
var count TokenCount
count.Add(text)
return count.words, count.punctuation
}
// TokenCount can be copied and extended with adjacent text fragments.
type TokenCount struct {
words, punctuation int
inWord bool
}
func (c *TokenCount) Add(text string) {
for _, r := range text {
nonspace := !unicode.IsSpace(r)
if nonspace && !inWord {
words++
if nonspace && !c.inWord {
c.words++
}
inWord = nonspace
c.inWord = nonspace
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
c.punctuation++
}
}
return words, punctuation
}
func (c TokenCount) Estimate() int {
return c.words + c.punctuation/2
}
// ConstTokenEstimator returns a constant token count for testing purposes.

View File

@@ -1,8 +1,11 @@
package normalization
import (
"slices"
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
@@ -30,177 +33,88 @@ type NormalizationSummary struct {
// NormalizeTranscript performs deterministic normalization on a transcript
type NormalizeTranscript struct {
config NormalizationConfig
estimator *SimpleTokenEstimator
config NormalizationConfig
}
// NewNormalizer creates a new transcript normalizer with the given configuration
// NewNormalizer creates a transcript normalizer with the given configuration.
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
return &NormalizeTranscript{
config: config,
estimator: &SimpleTokenEstimator{},
}
return &NormalizeTranscript{config: config}
}
// Normalize performs deterministic normalization on the given transcript
// Normalize performs deterministic normalization on the given transcript.
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
summary := &NormalizationSummary{
InputSegmentCount: len(transcript.Segments),
}
summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)}
if len(transcript.Segments) == 0 {
return transcript, summary
}
// Sort segments chronologically
sortedSegments := make([]schema.Segment, len(transcript.Segments))
copy(sortedSegments, transcript.Segments)
sort.Slice(sortedSegments, func(i, j int) bool {
return sortedSegments[i].Start < sortedSegments[j].Start
})
sorted := slices.Clone(transcript.Segments)
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
// Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment
var currentSegment schema.Segment
for i, segment := range sortedSegments {
if i == 0 {
// Initialize with first segment
currentSegment = segment
continue
var normalized []schema.Segment
var mergedText strings.Builder
var tokens chunking.TokenCount
current := sorted[0]
finishSegment := func() {
if mergedText.Len() > 0 {
current.Text = mergedText.String()
mergedText.Reset()
}
normalized = append(normalized, current)
tokens = chunking.TokenCount{}
}
// Check if we should merge with current segment
gap := segment.Start - currentSegment.End
merge, reason := shouldMergeWithReason(n.config, n.estimator, &currentSegment, &segment, gap)
if merge {
summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
} else {
// Track the reason for not merging
switch reason {
case "different_speakers":
summary.SkippedMerges.DifferentSpeakers++
case "gap_too_large":
summary.SkippedMerges.GapTooLarge++
case "duration_exceeded":
summary.SkippedMerges.DurationExceeded++
case "token_limit_exceeded":
summary.SkippedMerges.TokenLimitExceeded++
for _, next := range sorted[1:] {
gap := next.Start - current.End
end := max(current.End, next.End)
switch {
case current.Speaker != next.Speaker:
summary.SkippedMerges.DifferentSpeakers++
case gap > n.config.MaxSegmentGap:
summary.SkippedMerges.GapTooLarge++
case end-current.Start > n.config.MaxSegmentDuration:
summary.SkippedMerges.DurationExceeded++
default:
joiner := " "
if gap >= n.config.EllipsisGap {
joiner = "... "
}
// Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment
estimate := tokens
if mergedText.Len() == 0 {
estimate.Add(current.Text)
}
estimate.Add(joiner)
estimate.Add(next.Text)
if estimate.Estimate() <= n.config.MaxSegmentTokens {
if mergedText.Len() == 0 {
mergedText.WriteString(current.Text)
// Own the category buffer before adding to it.
owned := make([]string, 0, len(current.Categories)+len(next.Categories))
current.Categories = append(owned, current.Categories...)
}
mergedText.WriteString(joiner)
mergedText.WriteString(next.Text)
current.Categories = append(current.Categories, next.Categories...)
current.End = end
tokens = estimate
summary.MergesPerformed++
continue
}
summary.SkippedMerges.TokenLimitExceeded++
}
finishSegment()
current = next
}
if current.Text != "" || mergedText.Len() > 0 {
finishSegment()
}
// Add the last segment
if len(currentSegment.Text) > 0 {
normalizedSegments = append(normalizedSegments, currentSegment)
}
// Reassign sequential IDs starting at 1
for i, segment := range normalizedSegments {
if segment.ID != i+1 {
for i := range normalized {
if normalized[i].ID != i+1 {
summary.IDsReassigned++
}
normalizedSegments[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalizedSegments)
return &schema.Transcript{Segments: normalizedSegments}, summary
}
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) bool {
// Don't merge if different speakers
if current.Speaker != next.Speaker {
return false
}
// Don't merge if gap is too large
if gap > config.MaxSegmentGap {
return false
}
// Calculate what the merged segment would look like
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
// Don't merge if duration would be exceeded
if mergedDuration > config.MaxSegmentDuration {
return false
}
// Don't merge if token estimate would be exceeded
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false
}
return true
}
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) (bool, string) {
if current.Speaker != next.Speaker {
return false, "different_speakers"
}
if gap > config.MaxSegmentGap {
return false, "gap_too_large"
}
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
if mergedDuration > config.MaxSegmentDuration {
return false, "duration_exceeded"
}
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false, "token_limit_exceeded"
}
return true, ""
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text
if gap >= ellipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
// Merge categories
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
categories = append(categories, current.Categories...)
categories = append(categories, next.Categories...)
return schema.Segment{
ID: current.ID, // Will be reassigned later
Speaker: current.Speaker,
Start: current.Start,
End: next.End,
Text: mergedText,
Categories: categories,
normalized[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalized)
return &schema.Transcript{Segments: normalized}, summary
}

View File

@@ -3,45 +3,13 @@ package normalization
import (
"encoding/json"
"os"
"slices"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
func TestNormalizationNoOp(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should be no-op since different speakers
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.InputSegmentCount != 2 {
t.Errorf("expected input count 2, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 2 {
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 0 {
t.Errorf("expected 0 merges, got %d", summary.MergesPerformed)
}
}
func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
@@ -121,6 +89,9 @@ func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) {
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.InputSegmentCount != 2 || summary.OutputSegmentCount != 2 || summary.MergesPerformed != 0 {
t.Fatalf("no-merge summary: %+v", summary)
}
if summary.SkippedMerges.DifferentSpeakers != 1 {
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
}
@@ -557,3 +528,72 @@ func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
}
}
}
func TestNormalizeKeepsTheEndOfNestedSegments(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{Speaker: "A", Start: 0, End: 8, Text: "outer", Categories: []string{"first", "shared"}},
{Speaker: "A", Start: 1, End: 2, Text: "inner", Categories: []string{"shared"}},
{Speaker: "A", Start: 8.5, End: 9, Text: "following", Categories: []string{"last"}},
{Speaker: "A", Start: 9.5, End: 11, Text: "too long"},
}}
config := NormalizationConfig{MaxSegmentGap: 1, EllipsisGap: 0.5, MaxSegmentDuration: 10, MaxSegmentTokens: 100}
got, summary := NewNormalizer(config).Normalize(transcript)
if len(got.Segments) != 2 || summary.MergesPerformed != 2 || summary.SkippedMerges.DurationExceeded != 1 {
t.Fatalf("segments: %+v; summary: %+v", got.Segments, summary)
}
if s := got.Segments[0]; s.Start != 0 || s.End != 9 || s.Text != "outer inner... following" ||
!slices.Equal(s.Categories, []string{"first", "shared", "shared", "last"}) {
t.Fatalf("merged segment: %+v", s)
}
config.MaxSegmentDuration = 5
got, summary = NewNormalizer(config).Normalize(&schema.Transcript{Segments: transcript.Segments[:2]})
if len(got.Segments) != 2 || summary.SkippedMerges.DurationExceeded != 1 {
t.Fatalf("long outer interval did not constrain merge: %+v, %+v", got, summary)
}
}
func TestNormalizePreservesInputOrderAtEqualStarts(t *testing.T) {
var input []schema.Segment
for i := range 14 {
start := float64((i + 1) % 2)
input = append(input, schema.Segment{Speaker: string(rune('A' + i)), Start: start, End: start + 1, Text: "word"})
}
got, _ := NewNormalizer(NormalizationConfig{}).Normalize(&schema.Transcript{Segments: input})
var speakers strings.Builder
for _, segment := range got.Segments {
speakers.WriteString(segment.Speaker)
}
if names := speakers.String(); names != "BDFHJLNACEGIKM" {
t.Fatalf("wrong tied order: %q", names)
}
}
func TestNormalizeMergeUsesExactJoinedTokenEstimate(t *testing.T) {
for _, tc := range []struct {
first, second, want string
gap float64
tokens int
}{
{"A!", "B?", "A! B?", 0, 3},
{"A", "B?", "A... B?", 1, 4},
{"A\u2003", "B?", "A\u2003... B?", 1, 5},
} {
t.Run(tc.want, func(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{Speaker: "A", Start: 0, End: 1, Text: tc.first},
{Speaker: "A", Start: 1 + tc.gap, End: 3, Text: tc.second},
}}
config := NormalizationConfig{MaxSegmentGap: 2, EllipsisGap: 1, MaxSegmentDuration: 5, MaxSegmentTokens: tc.tokens}
got, _ := NewNormalizer(config).Normalize(transcript)
if len(got.Segments) != 1 || got.Segments[0].Text != tc.want {
t.Fatalf("expected merge at budget %d: %+v", tc.tokens, got)
}
config.MaxSegmentTokens--
got, summary := NewNormalizer(config).Normalize(transcript)
if len(got.Segments) != 2 || summary.SkippedMerges.TokenLimitExceeded != 1 {
t.Fatalf("merged above budget %d: %+v, %+v", config.MaxSegmentTokens, got, summary)
}
})
}
}

View File

@@ -1,33 +0,0 @@
package normalization
import (
"strings"
"unicode"
)
// SimpleTokenEstimator provides a basic deterministic token estimation.
// This is a placeholder that can be replaced with a more sophisticated estimator later.
type SimpleTokenEstimator struct{}
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
if text == "" {
return 0
}
// Simple heuristic: split on whitespace and count non-empty segments
words := strings.Fields(text)
tokenCount := len(words)
// Add some estimate for punctuation that might be separate tokens
punctuationCount := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuationCount++
}
}
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
return tokenCount + (punctuationCount / 2)
}