package normalization import ( "slices" "sort" "strings" "gitea.maximumdirect.net/eric/audita/v4/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v4/internal/core/schema" ) // NormalizationConfig holds configuration parameters for transcript normalization type NormalizationConfig struct { MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds) EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds) MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds) MaxSegmentTokens int // Maximum token estimate for a merged segment } // NormalizationSummary records the results and statistics of normalization type NormalizationSummary struct { InputSegmentCount int `json:"input_segment_count"` OutputSegmentCount int `json:"output_segment_count"` MergesPerformed int `json:"merges_performed"` IDsReassigned int `json:"ids_reassigned"` SkippedMerges struct { DifferentSpeakers int `json:"different_speakers"` GapTooLarge int `json:"gap_too_large"` DurationExceeded int `json:"duration_exceeded"` TokenLimitExceeded int `json:"token_limit_exceeded"` } `json:"skipped_merges"` } // NormalizeTranscript performs deterministic normalization on a transcript type NormalizeTranscript struct { config NormalizationConfig } // NewNormalizer creates a transcript normalizer with the given configuration. func NewNormalizer(config NormalizationConfig) *NormalizeTranscript { return &NormalizeTranscript{config: config} } // Normalize performs deterministic normalization on the given transcript. func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) { summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)} if len(transcript.Segments) == 0 { return transcript, summary } sorted := slices.Clone(transcript.Segments) sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start }) var normalized []schema.Segment var mergedText strings.Builder var tokens chunking.TokenCount current := sorted[0] finishSegment := func() { if mergedText.Len() > 0 { current.Text = mergedText.String() mergedText.Reset() } normalized = append(normalized, current) tokens = chunking.TokenCount{} } for _, next := range sorted[1:] { gap := next.Start - current.End end := max(current.End, next.End) switch { case current.Speaker != next.Speaker: summary.SkippedMerges.DifferentSpeakers++ case gap > n.config.MaxSegmentGap: summary.SkippedMerges.GapTooLarge++ case end-current.Start > n.config.MaxSegmentDuration: summary.SkippedMerges.DurationExceeded++ default: joiner := " " if gap >= n.config.EllipsisGap { joiner = "... " } estimate := tokens if mergedText.Len() == 0 { estimate.Add(current.Text) } estimate.Add(joiner) estimate.Add(next.Text) if estimate.Estimate() <= n.config.MaxSegmentTokens { if mergedText.Len() == 0 { mergedText.WriteString(current.Text) // Own the category buffer before adding to it. owned := make([]string, 0, len(current.Categories)+len(next.Categories)) current.Categories = append(owned, current.Categories...) } mergedText.WriteString(joiner) mergedText.WriteString(next.Text) current.Categories = append(current.Categories, next.Categories...) current.End = end tokens = estimate summary.MergesPerformed++ continue } summary.SkippedMerges.TokenLimitExceeded++ } finishSegment() current = next } if current.Text != "" || mergedText.Len() > 0 { finishSegment() } for i := range normalized { if normalized[i].ID != i+1 { summary.IDsReassigned++ } normalized[i].ID = i + 1 } summary.OutputSegmentCount = len(normalized) return &schema.Transcript{Segments: normalized}, summary }