Normalize nested intervals and build merges incrementally

This commit is contained in:
2026-11-16 03:59:44 +00:00
parent f465496dd8
commit 27f494d4e4
6 changed files with 173 additions and 229 deletions

View File

@@ -1,8 +1,11 @@
package normalization
import (
"slices"
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
@@ -30,177 +33,88 @@ type NormalizationSummary struct {
// NormalizeTranscript performs deterministic normalization on a transcript
type NormalizeTranscript struct {
config NormalizationConfig
estimator *SimpleTokenEstimator
config NormalizationConfig
}
// NewNormalizer creates a new transcript normalizer with the given configuration
// NewNormalizer creates a transcript normalizer with the given configuration.
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
return &NormalizeTranscript{
config: config,
estimator: &SimpleTokenEstimator{},
}
return &NormalizeTranscript{config: config}
}
// Normalize performs deterministic normalization on the given transcript
// Normalize performs deterministic normalization on the given transcript.
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
summary := &NormalizationSummary{
InputSegmentCount: len(transcript.Segments),
}
summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)}
if len(transcript.Segments) == 0 {
return transcript, summary
}
// Sort segments chronologically
sortedSegments := make([]schema.Segment, len(transcript.Segments))
copy(sortedSegments, transcript.Segments)
sort.Slice(sortedSegments, func(i, j int) bool {
return sortedSegments[i].Start < sortedSegments[j].Start
})
sorted := slices.Clone(transcript.Segments)
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
// Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment
var currentSegment schema.Segment
for i, segment := range sortedSegments {
if i == 0 {
// Initialize with first segment
currentSegment = segment
continue
var normalized []schema.Segment
var mergedText strings.Builder
var tokens chunking.TokenCount
current := sorted[0]
finishSegment := func() {
if mergedText.Len() > 0 {
current.Text = mergedText.String()
mergedText.Reset()
}
normalized = append(normalized, current)
tokens = chunking.TokenCount{}
}
// Check if we should merge with current segment
gap := segment.Start - currentSegment.End
merge, reason := shouldMergeWithReason(n.config, n.estimator, &currentSegment, &segment, gap)
if merge {
summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
} else {
// Track the reason for not merging
switch reason {
case "different_speakers":
summary.SkippedMerges.DifferentSpeakers++
case "gap_too_large":
summary.SkippedMerges.GapTooLarge++
case "duration_exceeded":
summary.SkippedMerges.DurationExceeded++
case "token_limit_exceeded":
summary.SkippedMerges.TokenLimitExceeded++
for _, next := range sorted[1:] {
gap := next.Start - current.End
end := max(current.End, next.End)
switch {
case current.Speaker != next.Speaker:
summary.SkippedMerges.DifferentSpeakers++
case gap > n.config.MaxSegmentGap:
summary.SkippedMerges.GapTooLarge++
case end-current.Start > n.config.MaxSegmentDuration:
summary.SkippedMerges.DurationExceeded++
default:
joiner := " "
if gap >= n.config.EllipsisGap {
joiner = "... "
}
// Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment
estimate := tokens
if mergedText.Len() == 0 {
estimate.Add(current.Text)
}
estimate.Add(joiner)
estimate.Add(next.Text)
if estimate.Estimate() <= n.config.MaxSegmentTokens {
if mergedText.Len() == 0 {
mergedText.WriteString(current.Text)
// Own the category buffer before adding to it.
owned := make([]string, 0, len(current.Categories)+len(next.Categories))
current.Categories = append(owned, current.Categories...)
}
mergedText.WriteString(joiner)
mergedText.WriteString(next.Text)
current.Categories = append(current.Categories, next.Categories...)
current.End = end
tokens = estimate
summary.MergesPerformed++
continue
}
summary.SkippedMerges.TokenLimitExceeded++
}
finishSegment()
current = next
}
if current.Text != "" || mergedText.Len() > 0 {
finishSegment()
}
// Add the last segment
if len(currentSegment.Text) > 0 {
normalizedSegments = append(normalizedSegments, currentSegment)
}
// Reassign sequential IDs starting at 1
for i, segment := range normalizedSegments {
if segment.ID != i+1 {
for i := range normalized {
if normalized[i].ID != i+1 {
summary.IDsReassigned++
}
normalizedSegments[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalizedSegments)
return &schema.Transcript{Segments: normalizedSegments}, summary
}
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) bool {
// Don't merge if different speakers
if current.Speaker != next.Speaker {
return false
}
// Don't merge if gap is too large
if gap > config.MaxSegmentGap {
return false
}
// Calculate what the merged segment would look like
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
// Don't merge if duration would be exceeded
if mergedDuration > config.MaxSegmentDuration {
return false
}
// Don't merge if token estimate would be exceeded
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false
}
return true
}
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) (bool, string) {
if current.Speaker != next.Speaker {
return false, "different_speakers"
}
if gap > config.MaxSegmentGap {
return false, "gap_too_large"
}
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
if mergedDuration > config.MaxSegmentDuration {
return false, "duration_exceeded"
}
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false, "token_limit_exceeded"
}
return true, ""
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text
if gap >= ellipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
// Merge categories
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
categories = append(categories, current.Categories...)
categories = append(categories, next.Categories...)
return schema.Segment{
ID: current.ID, // Will be reassigned later
Speaker: current.Speaker,
Start: current.Start,
End: next.End,
Text: mergedText,
Categories: categories,
normalized[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalized)
return &schema.Transcript{Segments: normalized}, summary
}