Normalize nested intervals and build merges incrementally
This commit is contained in:
@@ -1,8 +1,11 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
@@ -30,177 +33,88 @@ type NormalizationSummary struct {
|
||||
|
||||
// NormalizeTranscript performs deterministic normalization on a transcript
|
||||
type NormalizeTranscript struct {
|
||||
config NormalizationConfig
|
||||
estimator *SimpleTokenEstimator
|
||||
config NormalizationConfig
|
||||
}
|
||||
|
||||
// NewNormalizer creates a new transcript normalizer with the given configuration
|
||||
// NewNormalizer creates a transcript normalizer with the given configuration.
|
||||
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
|
||||
return &NormalizeTranscript{
|
||||
config: config,
|
||||
estimator: &SimpleTokenEstimator{},
|
||||
}
|
||||
return &NormalizeTranscript{config: config}
|
||||
}
|
||||
|
||||
// Normalize performs deterministic normalization on the given transcript
|
||||
// Normalize performs deterministic normalization on the given transcript.
|
||||
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
|
||||
summary := &NormalizationSummary{
|
||||
InputSegmentCount: len(transcript.Segments),
|
||||
}
|
||||
|
||||
summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)}
|
||||
if len(transcript.Segments) == 0 {
|
||||
return transcript, summary
|
||||
}
|
||||
|
||||
// Sort segments chronologically
|
||||
sortedSegments := make([]schema.Segment, len(transcript.Segments))
|
||||
copy(sortedSegments, transcript.Segments)
|
||||
sort.Slice(sortedSegments, func(i, j int) bool {
|
||||
return sortedSegments[i].Start < sortedSegments[j].Start
|
||||
})
|
||||
sorted := slices.Clone(transcript.Segments)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
|
||||
|
||||
// Merge same-speaker adjacent segments
|
||||
var normalizedSegments []schema.Segment
|
||||
var currentSegment schema.Segment
|
||||
|
||||
for i, segment := range sortedSegments {
|
||||
if i == 0 {
|
||||
// Initialize with first segment
|
||||
currentSegment = segment
|
||||
continue
|
||||
var normalized []schema.Segment
|
||||
var mergedText strings.Builder
|
||||
var tokens chunking.TokenCount
|
||||
current := sorted[0]
|
||||
finishSegment := func() {
|
||||
if mergedText.Len() > 0 {
|
||||
current.Text = mergedText.String()
|
||||
mergedText.Reset()
|
||||
}
|
||||
normalized = append(normalized, current)
|
||||
tokens = chunking.TokenCount{}
|
||||
}
|
||||
|
||||
// Check if we should merge with current segment
|
||||
gap := segment.Start - currentSegment.End
|
||||
merge, reason := shouldMergeWithReason(n.config, n.estimator, ¤tSegment, &segment, gap)
|
||||
if merge {
|
||||
summary.MergesPerformed++
|
||||
currentSegment = mergeSegments(¤tSegment, &segment, gap, n.config.EllipsisGap)
|
||||
} else {
|
||||
// Track the reason for not merging
|
||||
switch reason {
|
||||
case "different_speakers":
|
||||
summary.SkippedMerges.DifferentSpeakers++
|
||||
case "gap_too_large":
|
||||
summary.SkippedMerges.GapTooLarge++
|
||||
case "duration_exceeded":
|
||||
summary.SkippedMerges.DurationExceeded++
|
||||
case "token_limit_exceeded":
|
||||
summary.SkippedMerges.TokenLimitExceeded++
|
||||
for _, next := range sorted[1:] {
|
||||
gap := next.Start - current.End
|
||||
end := max(current.End, next.End)
|
||||
switch {
|
||||
case current.Speaker != next.Speaker:
|
||||
summary.SkippedMerges.DifferentSpeakers++
|
||||
case gap > n.config.MaxSegmentGap:
|
||||
summary.SkippedMerges.GapTooLarge++
|
||||
case end-current.Start > n.config.MaxSegmentDuration:
|
||||
summary.SkippedMerges.DurationExceeded++
|
||||
default:
|
||||
joiner := " "
|
||||
if gap >= n.config.EllipsisGap {
|
||||
joiner = "... "
|
||||
}
|
||||
// Finalize current segment and start new one
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
currentSegment = segment
|
||||
estimate := tokens
|
||||
if mergedText.Len() == 0 {
|
||||
estimate.Add(current.Text)
|
||||
}
|
||||
estimate.Add(joiner)
|
||||
estimate.Add(next.Text)
|
||||
if estimate.Estimate() <= n.config.MaxSegmentTokens {
|
||||
if mergedText.Len() == 0 {
|
||||
mergedText.WriteString(current.Text)
|
||||
// Own the category buffer before adding to it.
|
||||
owned := make([]string, 0, len(current.Categories)+len(next.Categories))
|
||||
current.Categories = append(owned, current.Categories...)
|
||||
}
|
||||
mergedText.WriteString(joiner)
|
||||
mergedText.WriteString(next.Text)
|
||||
current.Categories = append(current.Categories, next.Categories...)
|
||||
current.End = end
|
||||
tokens = estimate
|
||||
summary.MergesPerformed++
|
||||
continue
|
||||
}
|
||||
summary.SkippedMerges.TokenLimitExceeded++
|
||||
}
|
||||
finishSegment()
|
||||
current = next
|
||||
}
|
||||
if current.Text != "" || mergedText.Len() > 0 {
|
||||
finishSegment()
|
||||
}
|
||||
|
||||
// Add the last segment
|
||||
if len(currentSegment.Text) > 0 {
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
}
|
||||
|
||||
// Reassign sequential IDs starting at 1
|
||||
for i, segment := range normalizedSegments {
|
||||
if segment.ID != i+1 {
|
||||
for i := range normalized {
|
||||
if normalized[i].ID != i+1 {
|
||||
summary.IDsReassigned++
|
||||
}
|
||||
normalizedSegments[i].ID = i + 1
|
||||
}
|
||||
|
||||
summary.OutputSegmentCount = len(normalizedSegments)
|
||||
|
||||
return &schema.Transcript{Segments: normalizedSegments}, summary
|
||||
}
|
||||
|
||||
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
|
||||
current, next *schema.Segment, gap float64) bool {
|
||||
|
||||
// Don't merge if different speakers
|
||||
if current.Speaker != next.Speaker {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if gap is too large
|
||||
if gap > config.MaxSegmentGap {
|
||||
return false
|
||||
}
|
||||
|
||||
// Calculate what the merged segment would look like
|
||||
mergedText := current.Text
|
||||
if gap >= config.EllipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
mergedDuration := next.End - current.Start
|
||||
|
||||
// Don't merge if duration would be exceeded
|
||||
if mergedDuration > config.MaxSegmentDuration {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if token estimate would be exceeded
|
||||
tokenEstimate := estimator.EstimateTokens(mergedText)
|
||||
if tokenEstimate > config.MaxSegmentTokens {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
|
||||
current, next *schema.Segment, gap float64) (bool, string) {
|
||||
|
||||
if current.Speaker != next.Speaker {
|
||||
return false, "different_speakers"
|
||||
}
|
||||
if gap > config.MaxSegmentGap {
|
||||
return false, "gap_too_large"
|
||||
}
|
||||
|
||||
mergedText := current.Text
|
||||
if gap >= config.EllipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
mergedDuration := next.End - current.Start
|
||||
if mergedDuration > config.MaxSegmentDuration {
|
||||
return false, "duration_exceeded"
|
||||
}
|
||||
|
||||
tokenEstimate := estimator.EstimateTokens(mergedText)
|
||||
if tokenEstimate > config.MaxSegmentTokens {
|
||||
return false, "token_limit_exceeded"
|
||||
}
|
||||
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
|
||||
mergedText := current.Text
|
||||
if gap >= ellipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
// Merge categories
|
||||
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
|
||||
categories = append(categories, current.Categories...)
|
||||
categories = append(categories, next.Categories...)
|
||||
|
||||
return schema.Segment{
|
||||
ID: current.ID, // Will be reassigned later
|
||||
Speaker: current.Speaker,
|
||||
Start: current.Start,
|
||||
End: next.End,
|
||||
Text: mergedText,
|
||||
Categories: categories,
|
||||
normalized[i].ID = i + 1
|
||||
}
|
||||
summary.OutputSegmentCount = len(normalized)
|
||||
return &schema.Transcript{Segments: normalized}, summary
|
||||
}
|
||||
|
||||
@@ -3,45 +3,13 @@ package normalization
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestNormalizationNoOp(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should be no-op since different speakers
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.InputSegmentCount != 2 {
|
||||
t.Errorf("expected input count 2, got %d", summary.InputSegmentCount)
|
||||
}
|
||||
if summary.OutputSegmentCount != 2 {
|
||||
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
|
||||
}
|
||||
if summary.MergesPerformed != 0 {
|
||||
t.Errorf("expected 0 merges, got %d", summary.MergesPerformed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
@@ -121,6 +89,9 @@ func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) {
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.InputSegmentCount != 2 || summary.OutputSegmentCount != 2 || summary.MergesPerformed != 0 {
|
||||
t.Fatalf("no-merge summary: %+v", summary)
|
||||
}
|
||||
if summary.SkippedMerges.DifferentSpeakers != 1 {
|
||||
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
|
||||
}
|
||||
@@ -557,3 +528,72 @@ func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeKeepsTheEndOfNestedSegments(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{Speaker: "A", Start: 0, End: 8, Text: "outer", Categories: []string{"first", "shared"}},
|
||||
{Speaker: "A", Start: 1, End: 2, Text: "inner", Categories: []string{"shared"}},
|
||||
{Speaker: "A", Start: 8.5, End: 9, Text: "following", Categories: []string{"last"}},
|
||||
{Speaker: "A", Start: 9.5, End: 11, Text: "too long"},
|
||||
}}
|
||||
config := NormalizationConfig{MaxSegmentGap: 1, EllipsisGap: 0.5, MaxSegmentDuration: 10, MaxSegmentTokens: 100}
|
||||
got, summary := NewNormalizer(config).Normalize(transcript)
|
||||
if len(got.Segments) != 2 || summary.MergesPerformed != 2 || summary.SkippedMerges.DurationExceeded != 1 {
|
||||
t.Fatalf("segments: %+v; summary: %+v", got.Segments, summary)
|
||||
}
|
||||
if s := got.Segments[0]; s.Start != 0 || s.End != 9 || s.Text != "outer inner... following" ||
|
||||
!slices.Equal(s.Categories, []string{"first", "shared", "shared", "last"}) {
|
||||
t.Fatalf("merged segment: %+v", s)
|
||||
}
|
||||
|
||||
config.MaxSegmentDuration = 5
|
||||
got, summary = NewNormalizer(config).Normalize(&schema.Transcript{Segments: transcript.Segments[:2]})
|
||||
if len(got.Segments) != 2 || summary.SkippedMerges.DurationExceeded != 1 {
|
||||
t.Fatalf("long outer interval did not constrain merge: %+v, %+v", got, summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePreservesInputOrderAtEqualStarts(t *testing.T) {
|
||||
var input []schema.Segment
|
||||
for i := range 14 {
|
||||
start := float64((i + 1) % 2)
|
||||
input = append(input, schema.Segment{Speaker: string(rune('A' + i)), Start: start, End: start + 1, Text: "word"})
|
||||
}
|
||||
got, _ := NewNormalizer(NormalizationConfig{}).Normalize(&schema.Transcript{Segments: input})
|
||||
var speakers strings.Builder
|
||||
for _, segment := range got.Segments {
|
||||
speakers.WriteString(segment.Speaker)
|
||||
}
|
||||
if names := speakers.String(); names != "BDFHJLNACEGIKM" {
|
||||
t.Fatalf("wrong tied order: %q", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMergeUsesExactJoinedTokenEstimate(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
first, second, want string
|
||||
gap float64
|
||||
tokens int
|
||||
}{
|
||||
{"A!", "B?", "A! B?", 0, 3},
|
||||
{"A", "B?", "A... B?", 1, 4},
|
||||
{"A\u2003", "B?", "A\u2003... B?", 1, 5},
|
||||
} {
|
||||
t.Run(tc.want, func(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{Speaker: "A", Start: 0, End: 1, Text: tc.first},
|
||||
{Speaker: "A", Start: 1 + tc.gap, End: 3, Text: tc.second},
|
||||
}}
|
||||
config := NormalizationConfig{MaxSegmentGap: 2, EllipsisGap: 1, MaxSegmentDuration: 5, MaxSegmentTokens: tc.tokens}
|
||||
got, _ := NewNormalizer(config).Normalize(transcript)
|
||||
if len(got.Segments) != 1 || got.Segments[0].Text != tc.want {
|
||||
t.Fatalf("expected merge at budget %d: %+v", tc.tokens, got)
|
||||
}
|
||||
config.MaxSegmentTokens--
|
||||
got, summary := NewNormalizer(config).Normalize(transcript)
|
||||
if len(got.Segments) != 2 || summary.SkippedMerges.TokenLimitExceeded != 1 {
|
||||
t.Fatalf("merged above budget %d: %+v, %+v", config.MaxSegmentTokens, got, summary)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
||||
// This is a placeholder that can be replaced with a more sophisticated estimator later.
|
||||
type SimpleTokenEstimator struct{}
|
||||
|
||||
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Simple heuristic: split on whitespace and count non-empty segments
|
||||
words := strings.Fields(text)
|
||||
tokenCount := len(words)
|
||||
|
||||
// Add some estimate for punctuation that might be separate tokens
|
||||
punctuationCount := 0
|
||||
for _, r := range text {
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuationCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
||||
return tokenCount + (punctuationCount / 2)
|
||||
}
|
||||
Reference in New Issue
Block a user