Normalize nested intervals and build merges incrementally

This commit is contained in:
2026-11-16 03:59:44 +00:00
parent f465496dd8
commit 27f494d4e4
6 changed files with 173 additions and 229 deletions

View File

@@ -7,8 +7,9 @@ formats; [configuration](../config.md) owns settings; the
1. CLI loads resolved configuration and transcript input. Preflight checks all
selected glossary views and roles before the output session prepares resources.
Transcript segments then enter canonical normalization.
2. Normalization sorts and merges segments using configured time and token
limits, reassigning IDs where needed.
2. Normalization sorts segments by start time and preserves input order for
ties. It evaluates merge limits using the later end time and the same token
estimate as chunking, then reassigns IDs where needed.
3. The runner partitions the normalized transcript exactly once before its
module loop. Initial section identity, segment membership, and token
estimates stay fixed while text evolves.

View File

@@ -245,6 +245,13 @@ Audit #17, #21, #35, #53, #59, #62 (merge helper).
nested intervals, ties, joiners, category ordering, and exact estimates.
Compare allocations or growth on a representative run of same-speaker merges.
**Complete.** Verified stable start-time ties, nested intervals, exact joiner
and token behavior, and category order. The merge uses incremental text,
categories, and the shared token estimator. On synthetic same-speaker runs
(Go 1.27.3, three iterations each), allocations fell from 238.1 MB to 0.42 MB
per operation for 2,000 segments and from 960.1 MB to 0.85 MB for 4,000. The
CLI build and full offline suite passed.
## Stage 7 — Make feasible explicit section targets succeed
Audit #20 and the infeasibility portion of #24.

View File

@@ -23,24 +23,39 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator {
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
words, punctuation := e.CountComponents(text)
return words + punctuation/2
var count TokenCount
count.Add(text)
return count.Estimate()
}
// CountComponents returns unrounded counts for budgets composed from fragments.
func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
inWord := false
var count TokenCount
count.Add(text)
return count.words, count.punctuation
}
// TokenCount can be copied and extended with adjacent text fragments.
type TokenCount struct {
words, punctuation int
inWord bool
}
func (c *TokenCount) Add(text string) {
for _, r := range text {
nonspace := !unicode.IsSpace(r)
if nonspace && !inWord {
words++
if nonspace && !c.inWord {
c.words++
}
inWord = nonspace
c.inWord = nonspace
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
c.punctuation++
}
}
return words, punctuation
}
func (c TokenCount) Estimate() int {
return c.words + c.punctuation/2
}
// ConstTokenEstimator returns a constant token count for testing purposes.

View File

@@ -1,8 +1,11 @@
package normalization
import (
"slices"
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
@@ -30,177 +33,88 @@ type NormalizationSummary struct {
// NormalizeTranscript performs deterministic normalization on a transcript
type NormalizeTranscript struct {
config NormalizationConfig
estimator *SimpleTokenEstimator
config NormalizationConfig
}
// NewNormalizer creates a new transcript normalizer with the given configuration
// NewNormalizer creates a transcript normalizer with the given configuration.
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
return &NormalizeTranscript{
config: config,
estimator: &SimpleTokenEstimator{},
}
return &NormalizeTranscript{config: config}
}
// Normalize performs deterministic normalization on the given transcript
// Normalize performs deterministic normalization on the given transcript.
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
summary := &NormalizationSummary{
InputSegmentCount: len(transcript.Segments),
}
summary := &NormalizationSummary{InputSegmentCount: len(transcript.Segments)}
if len(transcript.Segments) == 0 {
return transcript, summary
}
// Sort segments chronologically
sortedSegments := make([]schema.Segment, len(transcript.Segments))
copy(sortedSegments, transcript.Segments)
sort.Slice(sortedSegments, func(i, j int) bool {
return sortedSegments[i].Start < sortedSegments[j].Start
})
sorted := slices.Clone(transcript.Segments)
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
// Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment
var currentSegment schema.Segment
for i, segment := range sortedSegments {
if i == 0 {
// Initialize with first segment
currentSegment = segment
continue
var normalized []schema.Segment
var mergedText strings.Builder
var tokens chunking.TokenCount
current := sorted[0]
finishSegment := func() {
if mergedText.Len() > 0 {
current.Text = mergedText.String()
mergedText.Reset()
}
normalized = append(normalized, current)
tokens = chunking.TokenCount{}
}
// Check if we should merge with current segment
gap := segment.Start - currentSegment.End
merge, reason := shouldMergeWithReason(n.config, n.estimator, &currentSegment, &segment, gap)
if merge {
summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
} else {
// Track the reason for not merging
switch reason {
case "different_speakers":
summary.SkippedMerges.DifferentSpeakers++
case "gap_too_large":
summary.SkippedMerges.GapTooLarge++
case "duration_exceeded":
summary.SkippedMerges.DurationExceeded++
case "token_limit_exceeded":
summary.SkippedMerges.TokenLimitExceeded++
for _, next := range sorted[1:] {
gap := next.Start - current.End
end := max(current.End, next.End)
switch {
case current.Speaker != next.Speaker:
summary.SkippedMerges.DifferentSpeakers++
case gap > n.config.MaxSegmentGap:
summary.SkippedMerges.GapTooLarge++
case end-current.Start > n.config.MaxSegmentDuration:
summary.SkippedMerges.DurationExceeded++
default:
joiner := " "
if gap >= n.config.EllipsisGap {
joiner = "... "
}
// Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment
estimate := tokens
if mergedText.Len() == 0 {
estimate.Add(current.Text)
}
estimate.Add(joiner)
estimate.Add(next.Text)
if estimate.Estimate() <= n.config.MaxSegmentTokens {
if mergedText.Len() == 0 {
mergedText.WriteString(current.Text)
// Own the category buffer before adding to it.
owned := make([]string, 0, len(current.Categories)+len(next.Categories))
current.Categories = append(owned, current.Categories...)
}
mergedText.WriteString(joiner)
mergedText.WriteString(next.Text)
current.Categories = append(current.Categories, next.Categories...)
current.End = end
tokens = estimate
summary.MergesPerformed++
continue
}
summary.SkippedMerges.TokenLimitExceeded++
}
finishSegment()
current = next
}
if current.Text != "" || mergedText.Len() > 0 {
finishSegment()
}
// Add the last segment
if len(currentSegment.Text) > 0 {
normalizedSegments = append(normalizedSegments, currentSegment)
}
// Reassign sequential IDs starting at 1
for i, segment := range normalizedSegments {
if segment.ID != i+1 {
for i := range normalized {
if normalized[i].ID != i+1 {
summary.IDsReassigned++
}
normalizedSegments[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalizedSegments)
return &schema.Transcript{Segments: normalizedSegments}, summary
}
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) bool {
// Don't merge if different speakers
if current.Speaker != next.Speaker {
return false
}
// Don't merge if gap is too large
if gap > config.MaxSegmentGap {
return false
}
// Calculate what the merged segment would look like
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
// Don't merge if duration would be exceeded
if mergedDuration > config.MaxSegmentDuration {
return false
}
// Don't merge if token estimate would be exceeded
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false
}
return true
}
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) (bool, string) {
if current.Speaker != next.Speaker {
return false, "different_speakers"
}
if gap > config.MaxSegmentGap {
return false, "gap_too_large"
}
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
if mergedDuration > config.MaxSegmentDuration {
return false, "duration_exceeded"
}
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false, "token_limit_exceeded"
}
return true, ""
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text
if gap >= ellipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
// Merge categories
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
categories = append(categories, current.Categories...)
categories = append(categories, next.Categories...)
return schema.Segment{
ID: current.ID, // Will be reassigned later
Speaker: current.Speaker,
Start: current.Start,
End: next.End,
Text: mergedText,
Categories: categories,
normalized[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalized)
return &schema.Transcript{Segments: normalized}, summary
}

View File

@@ -3,45 +3,13 @@ package normalization
import (
"encoding/json"
"os"
"slices"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
func TestNormalizationNoOp(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should be no-op since different speakers
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.InputSegmentCount != 2 {
t.Errorf("expected input count 2, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 2 {
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 0 {
t.Errorf("expected 0 merges, got %d", summary.MergesPerformed)
}
}
func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
@@ -121,6 +89,9 @@ func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) {
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.InputSegmentCount != 2 || summary.OutputSegmentCount != 2 || summary.MergesPerformed != 0 {
t.Fatalf("no-merge summary: %+v", summary)
}
if summary.SkippedMerges.DifferentSpeakers != 1 {
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
}
@@ -557,3 +528,72 @@ func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
}
}
}
func TestNormalizeKeepsTheEndOfNestedSegments(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{Speaker: "A", Start: 0, End: 8, Text: "outer", Categories: []string{"first", "shared"}},
{Speaker: "A", Start: 1, End: 2, Text: "inner", Categories: []string{"shared"}},
{Speaker: "A", Start: 8.5, End: 9, Text: "following", Categories: []string{"last"}},
{Speaker: "A", Start: 9.5, End: 11, Text: "too long"},
}}
config := NormalizationConfig{MaxSegmentGap: 1, EllipsisGap: 0.5, MaxSegmentDuration: 10, MaxSegmentTokens: 100}
got, summary := NewNormalizer(config).Normalize(transcript)
if len(got.Segments) != 2 || summary.MergesPerformed != 2 || summary.SkippedMerges.DurationExceeded != 1 {
t.Fatalf("segments: %+v; summary: %+v", got.Segments, summary)
}
if s := got.Segments[0]; s.Start != 0 || s.End != 9 || s.Text != "outer inner... following" ||
!slices.Equal(s.Categories, []string{"first", "shared", "shared", "last"}) {
t.Fatalf("merged segment: %+v", s)
}
config.MaxSegmentDuration = 5
got, summary = NewNormalizer(config).Normalize(&schema.Transcript{Segments: transcript.Segments[:2]})
if len(got.Segments) != 2 || summary.SkippedMerges.DurationExceeded != 1 {
t.Fatalf("long outer interval did not constrain merge: %+v, %+v", got, summary)
}
}
func TestNormalizePreservesInputOrderAtEqualStarts(t *testing.T) {
var input []schema.Segment
for i := range 14 {
start := float64((i + 1) % 2)
input = append(input, schema.Segment{Speaker: string(rune('A' + i)), Start: start, End: start + 1, Text: "word"})
}
got, _ := NewNormalizer(NormalizationConfig{}).Normalize(&schema.Transcript{Segments: input})
var speakers strings.Builder
for _, segment := range got.Segments {
speakers.WriteString(segment.Speaker)
}
if names := speakers.String(); names != "BDFHJLNACEGIKM" {
t.Fatalf("wrong tied order: %q", names)
}
}
func TestNormalizeMergeUsesExactJoinedTokenEstimate(t *testing.T) {
for _, tc := range []struct {
first, second, want string
gap float64
tokens int
}{
{"A!", "B?", "A! B?", 0, 3},
{"A", "B?", "A... B?", 1, 4},
{"A\u2003", "B?", "A\u2003... B?", 1, 5},
} {
t.Run(tc.want, func(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{Speaker: "A", Start: 0, End: 1, Text: tc.first},
{Speaker: "A", Start: 1 + tc.gap, End: 3, Text: tc.second},
}}
config := NormalizationConfig{MaxSegmentGap: 2, EllipsisGap: 1, MaxSegmentDuration: 5, MaxSegmentTokens: tc.tokens}
got, _ := NewNormalizer(config).Normalize(transcript)
if len(got.Segments) != 1 || got.Segments[0].Text != tc.want {
t.Fatalf("expected merge at budget %d: %+v", tc.tokens, got)
}
config.MaxSegmentTokens--
got, summary := NewNormalizer(config).Normalize(transcript)
if len(got.Segments) != 2 || summary.SkippedMerges.TokenLimitExceeded != 1 {
t.Fatalf("merged above budget %d: %+v, %+v", config.MaxSegmentTokens, got, summary)
}
})
}
}

View File

@@ -1,33 +0,0 @@
package normalization
import (
"strings"
"unicode"
)
// SimpleTokenEstimator provides a basic deterministic token estimation.
// This is a placeholder that can be replaced with a more sophisticated estimator later.
type SimpleTokenEstimator struct{}
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
if text == "" {
return 0
}
// Simple heuristic: split on whitespace and count non-empty segments
words := strings.Fields(text)
tokenCount := len(words)
// Add some estimate for punctuation that might be separate tokens
punctuationCount := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuationCount++
}
}
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
return tokenCount + (punctuationCount / 2)
}