From 27f494d4e42a508e1763e8b9758b957f646175d6 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 16 Nov 2026 03:59:44 +0000 Subject: [PATCH] Normalize nested intervals and build merges incrementally --- docs/internal/pipeline.md | 5 +- docs/roadmap/implementation.md | 7 + internal/core/chunking/tokens.go | 31 ++- internal/core/normalization/normalize.go | 220 ++++++------------ internal/core/normalization/normalize_test.go | 106 ++++++--- internal/core/normalization/tokens.go | 33 --- 6 files changed, 173 insertions(+), 229 deletions(-) delete mode 100644 internal/core/normalization/tokens.go diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 2bfaf7f..2417b85 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -7,8 +7,9 @@ formats; [configuration](../config.md) owns settings; the 1. CLI loads resolved configuration and transcript input. Preflight checks all selected glossary views and roles before the output session prepares resources. Transcript segments then enter canonical normalization. -2. Normalization sorts and merges segments using configured time and token - limits, reassigning IDs where needed. +2. Normalization sorts segments by start time and preserves input order for + ties. It evaluates merge limits using the later end time and the same token + estimate as chunking, then reassigns IDs where needed. 3. The runner partitions the normalized transcript exactly once before its module loop. Initial section identity, segment membership, and token estimates stay fixed while text evolves. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index b9dd3fc..012d776 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -245,6 +245,13 @@ Audit #17, #21, #35, #53, #59, #62 (merge helper). nested intervals, ties, joiners, category ordering, and exact estimates. Compare allocations or growth on a representative run of same-speaker merges. +**Complete.** Verified stable start-time ties, nested intervals, exact joiner +and token behavior, and category order. The merge uses incremental text, +categories, and the shared token estimator. On synthetic same-speaker runs +(Go 1.27.3, three iterations each), allocations fell from 238.1 MB to 0.42 MB +per operation for 2,000 segments and from 960.1 MB to 0.85 MB for 4,000. The +CLI build and full offline suite passed. + ## Stage 7 — Make feasible explicit section targets succeed Audit #20 and the infeasibility portion of #24. diff --git a/internal/core/chunking/tokens.go b/internal/core/chunking/tokens.go index b65601b..b3087ac 100644 --- a/internal/core/chunking/tokens.go +++ b/internal/core/chunking/tokens.go @@ -23,24 +23,39 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator { // This implementation uses a simple heuristic: count words and punctuation as tokens. // The estimate is deterministic and stable for the same input text. func (e *SimpleTokenEstimator) EstimateTokens(text string) int { - words, punctuation := e.CountComponents(text) - return words + punctuation/2 + var count TokenCount + count.Add(text) + return count.Estimate() } // CountComponents returns unrounded counts for budgets composed from fragments. func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) { - inWord := false + var count TokenCount + count.Add(text) + return count.words, count.punctuation +} + +// TokenCount can be copied and extended with adjacent text fragments. +type TokenCount struct { + words, punctuation int + inWord bool +} + +func (c *TokenCount) Add(text string) { for _, r := range text { nonspace := !unicode.IsSpace(r) - if nonspace && !inWord { - words++ + if nonspace && !c.inWord { + c.words++ } - inWord = nonspace + c.inWord = nonspace if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' { - punctuation++ + c.punctuation++ } } - return words, punctuation +} + +func (c TokenCount) Estimate() int { + return c.words + c.punctuation/2 } // ConstTokenEstimator returns a constant token count for testing purposes. diff --git a/internal/core/normalization/normalize.go b/internal/core/normalization/normalize.go index ebcf530..b3eb2b9 100644 --- a/internal/core/normalization/normalize.go +++ b/internal/core/normalization/normalize.go @@ -1,8 +1,11 @@ package normalization import ( + "slices" "sort" + "strings" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) @@ -30,177 +33,88 @@ type NormalizationSummary struct { // NormalizeTranscript performs deterministic normalization on a transcript type NormalizeTranscript struct { - config NormalizationConfig - estimator *SimpleTokenEstimator + config NormalizationConfig } -// NewNormalizer creates a new transcript normalizer with the given configuration +// NewNormalizer creates a transcript normalizer with the given configuration. func NewNormalizer(config NormalizationConfig) *NormalizeTranscript { - return &NormalizeTranscript{ - config: config, - estimator: &SimpleTokenEstimator{}, - } + return &NormalizeTranscript{config: config} } -// Normalize performs deterministic normalization on the given transcript +// Normalize performs deterministic normalization on the given transcript. func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) { - summary := &NormalizationSummary{ - InputSegmentCount: len(transcript.Segments), - } - + summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)} if len(transcript.Segments) == 0 { return transcript, summary } - // Sort segments chronologically - sortedSegments := make([]schema.Segment, len(transcript.Segments)) - copy(sortedSegments, transcript.Segments) - sort.Slice(sortedSegments, func(i, j int) bool { - return sortedSegments[i].Start < sortedSegments[j].Start - }) + sorted := slices.Clone(transcript.Segments) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start }) - // Merge same-speaker adjacent segments - var normalizedSegments []schema.Segment - var currentSegment schema.Segment - - for i, segment := range sortedSegments { - if i == 0 { - // Initialize with first segment - currentSegment = segment - continue + var normalized []schema.Segment + var mergedText strings.Builder + var tokens chunking.TokenCount + current := sorted[0] + finishSegment := func() { + if mergedText.Len() > 0 { + current.Text = mergedText.String() + mergedText.Reset() } + normalized = append(normalized, current) + tokens = chunking.TokenCount{} + } - // Check if we should merge with current segment - gap := segment.Start - currentSegment.End - merge, reason := shouldMergeWithReason(n.config, n.estimator, ¤tSegment, &segment, gap) - if merge { - summary.MergesPerformed++ - currentSegment = mergeSegments(¤tSegment, &segment, gap, n.config.EllipsisGap) - } else { - // Track the reason for not merging - switch reason { - case "different_speakers": - summary.SkippedMerges.DifferentSpeakers++ - case "gap_too_large": - summary.SkippedMerges.GapTooLarge++ - case "duration_exceeded": - summary.SkippedMerges.DurationExceeded++ - case "token_limit_exceeded": - summary.SkippedMerges.TokenLimitExceeded++ + for _, next := range sorted[1:] { + gap := next.Start - current.End + end := max(current.End, next.End) + switch { + case current.Speaker != next.Speaker: + summary.SkippedMerges.DifferentSpeakers++ + case gap > n.config.MaxSegmentGap: + summary.SkippedMerges.GapTooLarge++ + case end-current.Start > n.config.MaxSegmentDuration: + summary.SkippedMerges.DurationExceeded++ + default: + joiner := " " + if gap >= n.config.EllipsisGap { + joiner = "... " } - // Finalize current segment and start new one - normalizedSegments = append(normalizedSegments, currentSegment) - currentSegment = segment + estimate := tokens + if mergedText.Len() == 0 { + estimate.Add(current.Text) + } + estimate.Add(joiner) + estimate.Add(next.Text) + if estimate.Estimate() <= n.config.MaxSegmentTokens { + if mergedText.Len() == 0 { + mergedText.WriteString(current.Text) + // Own the category buffer before adding to it. + owned := make([]string, 0, len(current.Categories)+len(next.Categories)) + current.Categories = append(owned, current.Categories...) + } + mergedText.WriteString(joiner) + mergedText.WriteString(next.Text) + current.Categories = append(current.Categories, next.Categories...) + current.End = end + tokens = estimate + summary.MergesPerformed++ + continue + } + summary.SkippedMerges.TokenLimitExceeded++ } + finishSegment() + current = next + } + if current.Text != "" || mergedText.Len() > 0 { + finishSegment() } - // Add the last segment - if len(currentSegment.Text) > 0 { - normalizedSegments = append(normalizedSegments, currentSegment) - } - - // Reassign sequential IDs starting at 1 - for i, segment := range normalizedSegments { - if segment.ID != i+1 { + for i := range normalized { + if normalized[i].ID != i+1 { summary.IDsReassigned++ } - normalizedSegments[i].ID = i + 1 - } - - summary.OutputSegmentCount = len(normalizedSegments) - - return &schema.Transcript{Segments: normalizedSegments}, summary -} - -func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator, - current, next *schema.Segment, gap float64) bool { - - // Don't merge if different speakers - if current.Speaker != next.Speaker { - return false - } - - // Don't merge if gap is too large - if gap > config.MaxSegmentGap { - return false - } - - // Calculate what the merged segment would look like - mergedText := current.Text - if gap >= config.EllipsisGap { - mergedText += "... " - } else { - mergedText += " " - } - mergedText += next.Text - - mergedDuration := next.End - current.Start - - // Don't merge if duration would be exceeded - if mergedDuration > config.MaxSegmentDuration { - return false - } - - // Don't merge if token estimate would be exceeded - tokenEstimate := estimator.EstimateTokens(mergedText) - if tokenEstimate > config.MaxSegmentTokens { - return false - } - - return true -} - -func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator, - current, next *schema.Segment, gap float64) (bool, string) { - - if current.Speaker != next.Speaker { - return false, "different_speakers" - } - if gap > config.MaxSegmentGap { - return false, "gap_too_large" - } - - mergedText := current.Text - if gap >= config.EllipsisGap { - mergedText += "... " - } else { - mergedText += " " - } - mergedText += next.Text - - mergedDuration := next.End - current.Start - if mergedDuration > config.MaxSegmentDuration { - return false, "duration_exceeded" - } - - tokenEstimate := estimator.EstimateTokens(mergedText) - if tokenEstimate > config.MaxSegmentTokens { - return false, "token_limit_exceeded" - } - - return true, "" -} - -func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment { - mergedText := current.Text - if gap >= ellipsisGap { - mergedText += "... " - } else { - mergedText += " " - } - mergedText += next.Text - - // Merge categories - categories := make([]string, 0, len(current.Categories)+len(next.Categories)) - categories = append(categories, current.Categories...) - categories = append(categories, next.Categories...) - - return schema.Segment{ - ID: current.ID, // Will be reassigned later - Speaker: current.Speaker, - Start: current.Start, - End: next.End, - Text: mergedText, - Categories: categories, + normalized[i].ID = i + 1 } + summary.OutputSegmentCount = len(normalized) + return &schema.Transcript{Segments: normalized}, summary } diff --git a/internal/core/normalization/normalize_test.go b/internal/core/normalization/normalize_test.go index 987f59d..1e4ce39 100644 --- a/internal/core/normalization/normalize_test.go +++ b/internal/core/normalization/normalize_test.go @@ -3,45 +3,13 @@ package normalization import ( "encoding/json" "os" + "slices" "strings" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) -func TestNormalizationNoOp(t *testing.T) { - config := NormalizationConfig{ - MaxSegmentGap: 2.0, - EllipsisGap: 1.0, - MaxSegmentDuration: 60.0, - MaxSegmentTokens: 100, - } - normalizer := NewNormalizer(config) - - transcript := &schema.Transcript{ - Segments: []schema.Segment{ - {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, - {ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"}, - }, - } - - normalized, summary := normalizer.Normalize(transcript) - - // Should be no-op since different speakers - if len(normalized.Segments) != 2 { - t.Errorf("expected 2 segments, got %d", len(normalized.Segments)) - } - if summary.InputSegmentCount != 2 { - t.Errorf("expected input count 2, got %d", summary.InputSegmentCount) - } - if summary.OutputSegmentCount != 2 { - t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount) - } - if summary.MergesPerformed != 0 { - t.Errorf("expected 0 merges, got %d", summary.MergesPerformed) - } -} - func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) { config := NormalizationConfig{ MaxSegmentGap: 2.0, @@ -121,6 +89,9 @@ func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) { if len(normalized.Segments) != 2 { t.Errorf("expected 2 segments, got %d", len(normalized.Segments)) } + if summary.InputSegmentCount != 2 || summary.OutputSegmentCount != 2 || summary.MergesPerformed != 0 { + t.Fatalf("no-merge summary: %+v", summary) + } if summary.SkippedMerges.DifferentSpeakers != 1 { t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers) } @@ -557,3 +528,72 @@ func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) { } } } + +func TestNormalizeKeepsTheEndOfNestedSegments(t *testing.T) { + transcript := &schema.Transcript{Segments: []schema.Segment{ + {Speaker: "A", Start: 0, End: 8, Text: "outer", Categories: []string{"first", "shared"}}, + {Speaker: "A", Start: 1, End: 2, Text: "inner", Categories: []string{"shared"}}, + {Speaker: "A", Start: 8.5, End: 9, Text: "following", Categories: []string{"last"}}, + {Speaker: "A", Start: 9.5, End: 11, Text: "too long"}, + }} + config := NormalizationConfig{MaxSegmentGap: 1, EllipsisGap: 0.5, MaxSegmentDuration: 10, MaxSegmentTokens: 100} + got, summary := NewNormalizer(config).Normalize(transcript) + if len(got.Segments) != 2 || summary.MergesPerformed != 2 || summary.SkippedMerges.DurationExceeded != 1 { + t.Fatalf("segments: %+v; summary: %+v", got.Segments, summary) + } + if s := got.Segments[0]; s.Start != 0 || s.End != 9 || s.Text != "outer inner... following" || + !slices.Equal(s.Categories, []string{"first", "shared", "shared", "last"}) { + t.Fatalf("merged segment: %+v", s) + } + + config.MaxSegmentDuration = 5 + got, summary = NewNormalizer(config).Normalize(&schema.Transcript{Segments: transcript.Segments[:2]}) + if len(got.Segments) != 2 || summary.SkippedMerges.DurationExceeded != 1 { + t.Fatalf("long outer interval did not constrain merge: %+v, %+v", got, summary) + } +} + +func TestNormalizePreservesInputOrderAtEqualStarts(t *testing.T) { + var input []schema.Segment + for i := range 14 { + start := float64((i + 1) % 2) + input = append(input, schema.Segment{Speaker: string(rune('A' + i)), Start: start, End: start + 1, Text: "word"}) + } + got, _ := NewNormalizer(NormalizationConfig{}).Normalize(&schema.Transcript{Segments: input}) + var speakers strings.Builder + for _, segment := range got.Segments { + speakers.WriteString(segment.Speaker) + } + if names := speakers.String(); names != "BDFHJLNACEGIKM" { + t.Fatalf("wrong tied order: %q", names) + } +} + +func TestNormalizeMergeUsesExactJoinedTokenEstimate(t *testing.T) { + for _, tc := range []struct { + first, second, want string + gap float64 + tokens int + }{ + {"A!", "B?", "A! B?", 0, 3}, + {"A", "B?", "A... B?", 1, 4}, + {"A\u2003", "B?", "A\u2003... B?", 1, 5}, + } { + t.Run(tc.want, func(t *testing.T) { + transcript := &schema.Transcript{Segments: []schema.Segment{ + {Speaker: "A", Start: 0, End: 1, Text: tc.first}, + {Speaker: "A", Start: 1 + tc.gap, End: 3, Text: tc.second}, + }} + config := NormalizationConfig{MaxSegmentGap: 2, EllipsisGap: 1, MaxSegmentDuration: 5, MaxSegmentTokens: tc.tokens} + got, _ := NewNormalizer(config).Normalize(transcript) + if len(got.Segments) != 1 || got.Segments[0].Text != tc.want { + t.Fatalf("expected merge at budget %d: %+v", tc.tokens, got) + } + config.MaxSegmentTokens-- + got, summary := NewNormalizer(config).Normalize(transcript) + if len(got.Segments) != 2 || summary.SkippedMerges.TokenLimitExceeded != 1 { + t.Fatalf("merged above budget %d: %+v, %+v", config.MaxSegmentTokens, got, summary) + } + }) + } +} diff --git a/internal/core/normalization/tokens.go b/internal/core/normalization/tokens.go deleted file mode 100644 index 8e69ed7..0000000 --- a/internal/core/normalization/tokens.go +++ /dev/null @@ -1,33 +0,0 @@ -package normalization - -import ( - "strings" - "unicode" -) - -// SimpleTokenEstimator provides a basic deterministic token estimation. -// This is a placeholder that can be replaced with a more sophisticated estimator later. -type SimpleTokenEstimator struct{} - -// EstimateTokens provides a rough estimate of the number of tokens in the given text. -// This implementation uses a simple heuristic: count words and punctuation as tokens. -func (e *SimpleTokenEstimator) EstimateTokens(text string) int { - if text == "" { - return 0 - } - - // Simple heuristic: split on whitespace and count non-empty segments - words := strings.Fields(text) - tokenCount := len(words) - - // Add some estimate for punctuation that might be separate tokens - punctuationCount := 0 - for _, r := range text { - if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' { - punctuationCount++ - } - } - - // Rough estimate: each word is a token, plus half the punctuation as separate tokens - return tokenCount + (punctuationCount / 2) -}