Files
audita/internal/core/normalization/normalize.go

121 lines
3.8 KiB
Go
Raw Normal View History

package normalization
import (
"slices"
"sort"
"strings"
2026-11-17 19:57:22 +00:00
"gitea.maximumdirect.net/eric/audita/v4/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
)
// NormalizationConfig holds configuration parameters for transcript normalization
type NormalizationConfig struct {
MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds)
EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds)
MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds)
MaxSegmentTokens int // Maximum token estimate for a merged segment
}
// NormalizationSummary records the results and statistics of normalization
type NormalizationSummary struct {
InputSegmentCount int `json:"input_segment_count"`
OutputSegmentCount int `json:"output_segment_count"`
MergesPerformed int `json:"merges_performed"`
IDsReassigned int `json:"ids_reassigned"`
SkippedMerges struct {
DifferentSpeakers int `json:"different_speakers"`
GapTooLarge int `json:"gap_too_large"`
DurationExceeded int `json:"duration_exceeded"`
TokenLimitExceeded int `json:"token_limit_exceeded"`
} `json:"skipped_merges"`
}
// NormalizeTranscript performs deterministic normalization on a transcript
type NormalizeTranscript struct {
config NormalizationConfig
}
// NewNormalizer creates a transcript normalizer with the given configuration.
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
return &NormalizeTranscript{config: config}
}
// Normalize performs deterministic normalization on the given transcript.
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)}
if len(transcript.Segments) == 0 {
return transcript, summary
}
sorted := slices.Clone(transcript.Segments)
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
var normalized []schema.Segment
var mergedText strings.Builder
var tokens chunking.TokenCount
current := sorted[0]
finishSegment := func() {
if mergedText.Len() > 0 {
current.Text = mergedText.String()
mergedText.Reset()
}
normalized = append(normalized, current)
tokens = chunking.TokenCount{}
}
for _, next := range sorted[1:] {
gap := next.Start - current.End
end := max(current.End, next.End)
switch {
case current.Speaker != next.Speaker:
summary.SkippedMerges.DifferentSpeakers++
case gap > n.config.MaxSegmentGap:
summary.SkippedMerges.GapTooLarge++
case end-current.Start > n.config.MaxSegmentDuration:
summary.SkippedMerges.DurationExceeded++
default:
joiner := " "
if gap >= n.config.EllipsisGap {
joiner = "... "
}
estimate := tokens
if mergedText.Len() == 0 {
estimate.Add(current.Text)
}
estimate.Add(joiner)
estimate.Add(next.Text)
if estimate.Estimate() <= n.config.MaxSegmentTokens {
if mergedText.Len() == 0 {
mergedText.WriteString(current.Text)
// Own the category buffer before adding to it.
owned := make([]string, 0, len(current.Categories)+len(next.Categories))
current.Categories = append(owned, current.Categories...)
}
mergedText.WriteString(joiner)
mergedText.WriteString(next.Text)
current.Categories = append(current.Categories, next.Categories...)
current.End = end
tokens = estimate
summary.MergesPerformed++
continue
}
summary.SkippedMerges.TokenLimitExceeded++
}
finishSegment()
current = next
}
if current.Text != "" || mergedText.Len() > 0 {
finishSegment()
}
for i := range normalized {
if normalized[i].ID != i+1 {
summary.IDsReassigned++
}
normalized[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalized)
return &schema.Transcript{Segments: normalized}, summary
}