Restore fast typed decoding for ordinary transcripts
This commit is contained in:
@@ -146,6 +146,12 @@ and processes case variants in the baseline key order. The decoder keeps
|
||||
numeric tokens intact. Schema tests cover superseded and effective type
|
||||
mismatches, both case-variant orders, and exact IDs in both envelopes.
|
||||
|
||||
The follow-up restores direct typed decoding for ordinary segment arrays.
|
||||
Arrays with conflicting or escaped keys use compatibility decoding where
|
||||
needed. Both envelopes were verified with mixed segments. On the same
|
||||
100,000-segment fixture, median parse time fell from about 0.83 to 0.30
|
||||
seconds and allocation fell from 162 to 78 MB per parse.
|
||||
|
||||
**Affected plan:** Stage 5, item 1.
|
||||
|
||||
**Evidence:** The baseline selected the final value of an exact duplicate JSON
|
||||
|
||||
140
internal/core/schema/source_members.go
Normal file
140
internal/core/schema/source_members.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func decodeSourceSegments(raw []byte) ([]SourceSegment, error) {
|
||||
if !needsMemberResolution(raw, 2) {
|
||||
var segments []SourceSegment
|
||||
err := json.Unmarshal(raw, &segments)
|
||||
return segments, err
|
||||
}
|
||||
var decoded []sourceSegmentWithAliases
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
segments := make([]SourceSegment, len(decoded))
|
||||
for i, segment := range decoded {
|
||||
segments[i] = SourceSegment(segment)
|
||||
}
|
||||
return segments, nil
|
||||
}
|
||||
|
||||
type sourceSegmentWithAliases SourceSegment
|
||||
|
||||
// Resolve repeated exact names before field conversion, then apply case
|
||||
// variants in the same key order as the original object round trip.
|
||||
func (s *sourceSegmentWithAliases) UnmarshalJSON(raw []byte) error {
|
||||
if !needsMemberResolution(raw, 1) {
|
||||
return json.Unmarshal(raw, (*SourceSegment)(s))
|
||||
}
|
||||
var members map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &members); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(members)) {
|
||||
var field any
|
||||
switch sourceMemberFor(key) {
|
||||
case memberID:
|
||||
field = &s.ID
|
||||
case memberSpeaker:
|
||||
field = &s.Speaker
|
||||
case memberStart:
|
||||
field = &s.Start
|
||||
case memberEnd:
|
||||
field = &s.End
|
||||
case memberText:
|
||||
field = &s.Text
|
||||
case memberCategories:
|
||||
field = &s.Categories
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(members[key], field); err != nil {
|
||||
return fmt.Errorf("field %q: %w", key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type sourceMember uint8
|
||||
|
||||
const (
|
||||
memberID sourceMember = 1 << iota
|
||||
memberSpeaker
|
||||
memberStart
|
||||
memberEnd
|
||||
memberText
|
||||
memberCategories
|
||||
)
|
||||
|
||||
func sourceMemberFor(key string) sourceMember {
|
||||
switch {
|
||||
case strings.EqualFold(key, "id"):
|
||||
return memberID
|
||||
case strings.EqualFold(key, "speaker"):
|
||||
return memberSpeaker
|
||||
case strings.EqualFold(key, "start"):
|
||||
return memberStart
|
||||
case strings.EqualFold(key, "end"):
|
||||
return memberEnd
|
||||
case strings.EqualFold(key, "text"):
|
||||
return memberText
|
||||
case strings.EqualFold(key, "categories"):
|
||||
return memberCategories
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// needsMemberResolution looks for repeated recognized keys in valid JSON.
|
||||
// segmentDepth locates members within the array or a single segment.
|
||||
// Escaped keys use the compatibility decoder to resolve their spelling.
|
||||
func needsMemberResolution(raw []byte, segmentDepth int) bool {
|
||||
depth := 0
|
||||
wantKey := false
|
||||
var seen sourceMember
|
||||
for i := 0; i < len(raw); i++ {
|
||||
switch raw[i] {
|
||||
case '{':
|
||||
depth++
|
||||
if depth == segmentDepth {
|
||||
seen, wantKey = 0, true
|
||||
}
|
||||
case '[':
|
||||
depth++
|
||||
case '}', ']':
|
||||
depth--
|
||||
case ',':
|
||||
if depth == segmentDepth {
|
||||
wantKey = true
|
||||
}
|
||||
case '"':
|
||||
start := i + 1
|
||||
escaped := false
|
||||
for i++; raw[i] != '"'; i++ {
|
||||
if raw[i] == '\\' {
|
||||
escaped = true
|
||||
i++
|
||||
}
|
||||
}
|
||||
if depth != segmentDepth || !wantKey {
|
||||
continue
|
||||
}
|
||||
wantKey = false
|
||||
if escaped {
|
||||
return true
|
||||
}
|
||||
member := sourceMemberFor(string(raw[start:i]))
|
||||
if seen&member != 0 {
|
||||
return true
|
||||
}
|
||||
seen |= member
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -4,10 +4,8 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
|
||||
)
|
||||
@@ -21,38 +19,6 @@ type SourceSegment struct {
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON decodes only the final value of each exact key, then applies
|
||||
// case variants in the same key order as the original object round trip.
|
||||
func (s *SourceSegment) UnmarshalJSON(raw []byte) error {
|
||||
var members map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &members); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(members)) {
|
||||
var field any
|
||||
switch {
|
||||
case strings.EqualFold(key, "id"):
|
||||
field = &s.ID
|
||||
case strings.EqualFold(key, "speaker"):
|
||||
field = &s.Speaker
|
||||
case strings.EqualFold(key, "start"):
|
||||
field = &s.Start
|
||||
case strings.EqualFold(key, "end"):
|
||||
field = &s.End
|
||||
case strings.EqualFold(key, "text"):
|
||||
field = &s.Text
|
||||
case strings.EqualFold(key, "categories"):
|
||||
field = &s.Categories
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(members[key], field); err != nil {
|
||||
return fmt.Errorf("field %q: %w", key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Segment struct {
|
||||
ID int `json:"id"`
|
||||
Speaker string `json:"speaker"`
|
||||
@@ -80,8 +46,8 @@ func ParseSourceTranscriptJSON(raw []byte) (*SourceTranscript, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var segments []SourceSegment
|
||||
if err := json.Unmarshal(segmentsRaw, &segments); err != nil {
|
||||
segments, err := decodeSourceSegments(segmentsRaw)
|
||||
if err != nil {
|
||||
return nil, &ParseError{Message: fmt.Sprintf("failed to parse segments: %v", err)}
|
||||
}
|
||||
if len(segments) == 0 {
|
||||
|
||||
@@ -366,6 +366,34 @@ func TestSourceTranscriptUsesEffectiveMemberValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceTranscriptCombinesOrdinaryAndAliasedMembers(t *testing.T) {
|
||||
id := "7"
|
||||
if strconv.IntSize == 64 {
|
||||
id = "9007199254740993"
|
||||
}
|
||||
wantID, _ := strconv.Atoi(id)
|
||||
segments := `[
|
||||
{"speaker":"A","text":"He said \"{hello, world}\".","id":SOURCE_ID,
|
||||
"extra":{"speaker":12,"TEXT":"metadata"}},
|
||||
{"speaker":17,"spea\u006ber":"B","text":"expected","TEXT":"other","categories":[]},
|
||||
{"speaker":"C","ſpeaker":"D","text":"last"}
|
||||
]`
|
||||
segments = strings.Replace(segments, "SOURCE_ID", id, 1)
|
||||
for _, raw := range []string{segments, `{"segments":` + segments + "}"} {
|
||||
transcript, err := ParseSourceTranscriptJSON([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := transcript.Canonical().Segments
|
||||
if len(got) != 3 || got[0].Speaker != "A" || got[0].ID != wantID ||
|
||||
got[0].Text != `He said "{hello, world}".` ||
|
||||
got[1].Speaker != "B" || got[1].Text != "expected" || got[1].Categories == nil ||
|
||||
got[2].Speaker != "D" || got[2].Text != "last" {
|
||||
t.Fatalf("unexpected segments: %+v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscriptClonePreservesPresenceAndIndependence(t *testing.T) {
|
||||
original := &Transcript{Segments: []Segment{
|
||||
{ID: 40, Text: "original", Categories: []string{"keep"}},
|
||||
|
||||
Reference in New Issue
Block a user