2026-05-11 00:12:19 +00:00
|
|
|
package normalization
|
|
|
|
|
|
|
|
|
|
import (
|
2026-11-16 03:59:44 +00:00
|
|
|
"slices"
|
2026-05-11 00:12:19 +00:00
|
|
|
"sort"
|
2026-11-16 03:59:44 +00:00
|
|
|
"strings"
|
2026-05-11 00:12:19 +00:00
|
|
|
|
2026-11-17 19:57:22 +00:00
|
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/chunking"
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
|
2026-05-11 00:12:19 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// NormalizationConfig holds configuration parameters for transcript normalization
|
|
|
|
|
type NormalizationConfig struct {
|
|
|
|
|
MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds)
|
|
|
|
|
EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds)
|
|
|
|
|
MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds)
|
|
|
|
|
MaxSegmentTokens int // Maximum token estimate for a merged segment
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// NormalizationSummary records the results and statistics of normalization
|
|
|
|
|
type NormalizationSummary struct {
|
|
|
|
|
InputSegmentCount int `json:"input_segment_count"`
|
|
|
|
|
OutputSegmentCount int `json:"output_segment_count"`
|
|
|
|
|
MergesPerformed int `json:"merges_performed"`
|
|
|
|
|
IDsReassigned int `json:"ids_reassigned"`
|
|
|
|
|
SkippedMerges struct {
|
|
|
|
|
DifferentSpeakers int `json:"different_speakers"`
|
|
|
|
|
GapTooLarge int `json:"gap_too_large"`
|
|
|
|
|
DurationExceeded int `json:"duration_exceeded"`
|
|
|
|
|
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
|
|
|
|
} `json:"skipped_merges"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// NormalizeTranscript performs deterministic normalization on a transcript
|
|
|
|
|
type NormalizeTranscript struct {
|
2026-11-16 03:59:44 +00:00
|
|
|
config NormalizationConfig
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:59:44 +00:00
|
|
|
// NewNormalizer creates a transcript normalizer with the given configuration.
|
2026-05-11 00:12:19 +00:00
|
|
|
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
|
2026-11-16 03:59:44 +00:00
|
|
|
return &NormalizeTranscript{config: config}
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:59:44 +00:00
|
|
|
// Normalize performs deterministic normalization on the given transcript.
|
2026-05-11 00:12:19 +00:00
|
|
|
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
|
2026-11-16 03:59:44 +00:00
|
|
|
summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)}
|
2026-05-11 00:12:19 +00:00
|
|
|
if len(transcript.Segments) == 0 {
|
|
|
|
|
return transcript, summary
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:59:44 +00:00
|
|
|
sorted := slices.Clone(transcript.Segments)
|
|
|
|
|
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
|
2026-05-11 00:12:19 +00:00
|
|
|
|
2026-11-16 03:59:44 +00:00
|
|
|
var normalized []schema.Segment
|
|
|
|
|
var mergedText strings.Builder
|
|
|
|
|
var tokens chunking.TokenCount
|
|
|
|
|
current := sorted[0]
|
|
|
|
|
finishSegment := func() {
|
|
|
|
|
if mergedText.Len() > 0 {
|
|
|
|
|
current.Text = mergedText.String()
|
|
|
|
|
mergedText.Reset()
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
2026-11-16 03:59:44 +00:00
|
|
|
normalized = append(normalized, current)
|
|
|
|
|
tokens = chunking.TokenCount{}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, next := range sorted[1:] {
|
|
|
|
|
gap := next.Start - current.End
|
|
|
|
|
end := max(current.End, next.End)
|
|
|
|
|
switch {
|
|
|
|
|
case current.Speaker != next.Speaker:
|
|
|
|
|
summary.SkippedMerges.DifferentSpeakers++
|
|
|
|
|
case gap > n.config.MaxSegmentGap:
|
|
|
|
|
summary.SkippedMerges.GapTooLarge++
|
|
|
|
|
case end-current.Start > n.config.MaxSegmentDuration:
|
|
|
|
|
summary.SkippedMerges.DurationExceeded++
|
|
|
|
|
default:
|
|
|
|
|
joiner := " "
|
|
|
|
|
if gap >= n.config.EllipsisGap {
|
|
|
|
|
joiner = "... "
|
|
|
|
|
}
|
|
|
|
|
estimate := tokens
|
|
|
|
|
if mergedText.Len() == 0 {
|
|
|
|
|
estimate.Add(current.Text)
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
2026-11-16 03:59:44 +00:00
|
|
|
estimate.Add(joiner)
|
|
|
|
|
estimate.Add(next.Text)
|
|
|
|
|
if estimate.Estimate() <= n.config.MaxSegmentTokens {
|
|
|
|
|
if mergedText.Len() == 0 {
|
|
|
|
|
mergedText.WriteString(current.Text)
|
|
|
|
|
// Own the category buffer before adding to it.
|
|
|
|
|
owned := make([]string, 0, len(current.Categories)+len(next.Categories))
|
|
|
|
|
current.Categories = append(owned, current.Categories...)
|
|
|
|
|
}
|
|
|
|
|
mergedText.WriteString(joiner)
|
|
|
|
|
mergedText.WriteString(next.Text)
|
|
|
|
|
current.Categories = append(current.Categories, next.Categories...)
|
|
|
|
|
current.End = end
|
|
|
|
|
tokens = estimate
|
|
|
|
|
summary.MergesPerformed++
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
summary.SkippedMerges.TokenLimitExceeded++
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
2026-11-16 03:59:44 +00:00
|
|
|
finishSegment()
|
|
|
|
|
current = next
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
2026-11-16 03:59:44 +00:00
|
|
|
if current.Text != "" || mergedText.Len() > 0 {
|
|
|
|
|
finishSegment()
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
|
|
|
|
|
2026-11-16 03:59:44 +00:00
|
|
|
for i := range normalized {
|
|
|
|
|
if normalized[i].ID != i+1 {
|
2026-05-11 00:12:19 +00:00
|
|
|
summary.IDsReassigned++
|
|
|
|
|
}
|
2026-11-16 03:59:44 +00:00
|
|
|
normalized[i].ID = i + 1
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|
2026-11-16 03:59:44 +00:00
|
|
|
summary.OutputSegmentCount = len(normalized)
|
|
|
|
|
return &schema.Transcript{Segments: normalized}, summary
|
2026-05-11 00:12:19 +00:00
|
|
|
}
|