Restore fast typed decoding for ordinary transcripts

This commit is contained in:
2026-11-17 12:57:47 +00:00
parent 19755e1d47
commit 3e107bedd1
4 changed files with 176 additions and 36 deletions

View File

@@ -146,6 +146,12 @@ and processes case variants in the baseline key order. The decoder keeps
numeric tokens intact. Schema tests cover superseded and effective type
mismatches, both case-variant orders, and exact IDs in both envelopes.
The follow-up restores direct typed decoding for ordinary segment arrays.
Arrays with conflicting or escaped keys use compatibility decoding where
needed. Both envelopes were verified with mixed segments. On the same
100,000-segment fixture, median parse time fell from about 0.83 to 0.30
seconds and allocation fell from 162 to 78 MB per parse.
**Affected plan:** Stage 5, item 1.
**Evidence:** The baseline selected the final value of an exact duplicate JSON

View File

@@ -0,0 +1,140 @@
package schema
import (
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
)
func decodeSourceSegments(raw []byte) ([]SourceSegment, error) {
if !needsMemberResolution(raw, 2) {
var segments []SourceSegment
err := json.Unmarshal(raw, &segments)
return segments, err
}
var decoded []sourceSegmentWithAliases
if err := json.Unmarshal(raw, &decoded); err != nil {
return nil, err
}
segments := make([]SourceSegment, len(decoded))
for i, segment := range decoded {
segments[i] = SourceSegment(segment)
}
return segments, nil
}
type sourceSegmentWithAliases SourceSegment
// Resolve repeated exact names before field conversion, then apply case
// variants in the same key order as the original object round trip.
func (s *sourceSegmentWithAliases) UnmarshalJSON(raw []byte) error {
if !needsMemberResolution(raw, 1) {
return json.Unmarshal(raw, (*SourceSegment)(s))
}
var members map[string]json.RawMessage
if err := json.Unmarshal(raw, &members); err != nil {
return err
}
for _, key := range slices.Sorted(maps.Keys(members)) {
var field any
switch sourceMemberFor(key) {
case memberID:
field = &s.ID
case memberSpeaker:
field = &s.Speaker
case memberStart:
field = &s.Start
case memberEnd:
field = &s.End
case memberText:
field = &s.Text
case memberCategories:
field = &s.Categories
default:
continue
}
if err := json.Unmarshal(members[key], field); err != nil {
return fmt.Errorf("field %q: %w", key, err)
}
}
return nil
}
type sourceMember uint8
const (
memberID sourceMember = 1 << iota
memberSpeaker
memberStart
memberEnd
memberText
memberCategories
)
func sourceMemberFor(key string) sourceMember {
switch {
case strings.EqualFold(key, "id"):
return memberID
case strings.EqualFold(key, "speaker"):
return memberSpeaker
case strings.EqualFold(key, "start"):
return memberStart
case strings.EqualFold(key, "end"):
return memberEnd
case strings.EqualFold(key, "text"):
return memberText
case strings.EqualFold(key, "categories"):
return memberCategories
}
return 0
}
// needsMemberResolution looks for repeated recognized keys in valid JSON.
// segmentDepth locates members within the array or a single segment.
// Escaped keys use the compatibility decoder to resolve their spelling.
func needsMemberResolution(raw []byte, segmentDepth int) bool {
depth := 0
wantKey := false
var seen sourceMember
for i := 0; i < len(raw); i++ {
switch raw[i] {
case '{':
depth++
if depth == segmentDepth {
seen, wantKey = 0, true
}
case '[':
depth++
case '}', ']':
depth--
case ',':
if depth == segmentDepth {
wantKey = true
}
case '"':
start := i + 1
escaped := false
for i++; raw[i] != '"'; i++ {
if raw[i] == '\\' {
escaped = true
i++
}
}
if depth != segmentDepth || !wantKey {
continue
}
wantKey = false
if escaped {
return true
}
member := sourceMemberFor(string(raw[start:i]))
if seen&member != 0 {
return true
}
seen |= member
}
}
return false
}

View File

@@ -4,10 +4,8 @@ import (
"bytes"
"encoding/json"
"fmt"
"maps"
"math"
"slices"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
)
@@ -21,38 +19,6 @@ type SourceSegment struct {
Categories []string `json:"categories,omitempty"`
}
// UnmarshalJSON decodes only the final value of each exact key, then applies
// case variants in the same key order as the original object round trip.
func (s *SourceSegment) UnmarshalJSON(raw []byte) error {
var members map[string]json.RawMessage
if err := json.Unmarshal(raw, &members); err != nil {
return err
}
for _, key := range slices.Sorted(maps.Keys(members)) {
var field any
switch {
case strings.EqualFold(key, "id"):
field = &s.ID
case strings.EqualFold(key, "speaker"):
field = &s.Speaker
case strings.EqualFold(key, "start"):
field = &s.Start
case strings.EqualFold(key, "end"):
field = &s.End
case strings.EqualFold(key, "text"):
field = &s.Text
case strings.EqualFold(key, "categories"):
field = &s.Categories
default:
continue
}
if err := json.Unmarshal(members[key], field); err != nil {
return fmt.Errorf("field %q: %w", key, err)
}
}
return nil
}
type Segment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
@@ -80,8 +46,8 @@ func ParseSourceTranscriptJSON(raw []byte) (*SourceTranscript, error) {
return nil, err
}
var segments []SourceSegment
if err := json.Unmarshal(segmentsRaw, &segments); err != nil {
segments, err := decodeSourceSegments(segmentsRaw)
if err != nil {
return nil, &ParseError{Message: fmt.Sprintf("failed to parse segments: %v", err)}
}
if len(segments) == 0 {

View File

@@ -366,6 +366,34 @@ func TestSourceTranscriptUsesEffectiveMemberValues(t *testing.T) {
}
}
func TestSourceTranscriptCombinesOrdinaryAndAliasedMembers(t *testing.T) {
id := "7"
if strconv.IntSize == 64 {
id = "9007199254740993"
}
wantID, _ := strconv.Atoi(id)
segments := `[
{"speaker":"A","text":"He said \"{hello, world}\".","id":SOURCE_ID,
"extra":{"speaker":12,"TEXT":"metadata"}},
{"speaker":17,"spea\u006ber":"B","text":"expected","TEXT":"other","categories":[]},
{"speaker":"C","ſpeaker":"D","text":"last"}
]`
segments = strings.Replace(segments, "SOURCE_ID", id, 1)
for _, raw := range []string{segments, `{"segments":` + segments + "}"} {
transcript, err := ParseSourceTranscriptJSON([]byte(raw))
if err != nil {
t.Fatal(err)
}
got := transcript.Canonical().Segments
if len(got) != 3 || got[0].Speaker != "A" || got[0].ID != wantID ||
got[0].Text != `He said "{hello, world}".` ||
got[1].Speaker != "B" || got[1].Text != "expected" || got[1].Categories == nil ||
got[2].Speaker != "D" || got[2].Text != "last" {
t.Fatalf("unexpected segments: %+v", got)
}
}
}
func TestTranscriptClonePreservesPresenceAndIndependence(t *testing.T) {
original := &Transcript{Segments: []Segment{
{ID: 40, Text: "original", Categories: []string{"keep"}},