diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 811f254..1ed72e1 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -146,6 +146,12 @@ and processes case variants in the baseline key order. The decoder keeps numeric tokens intact. Schema tests cover superseded and effective type mismatches, both case-variant orders, and exact IDs in both envelopes. +The follow-up restores direct typed decoding for ordinary segment arrays. +Arrays with conflicting or escaped keys use compatibility decoding where +needed. Both envelopes were verified with mixed segments. On the same +100,000-segment fixture, median parse time fell from about 0.83 to 0.30 +seconds and allocation fell from 162 to 78 MB per parse. + **Affected plan:** Stage 5, item 1. **Evidence:** The baseline selected the final value of an exact duplicate JSON diff --git a/internal/core/schema/source_members.go b/internal/core/schema/source_members.go new file mode 100644 index 0000000..d7aa025 --- /dev/null +++ b/internal/core/schema/source_members.go @@ -0,0 +1,140 @@ +package schema + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "strings" +) + +func decodeSourceSegments(raw []byte) ([]SourceSegment, error) { + if !needsMemberResolution(raw, 2) { + var segments []SourceSegment + err := json.Unmarshal(raw, &segments) + return segments, err + } + var decoded []sourceSegmentWithAliases + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, err + } + segments := make([]SourceSegment, len(decoded)) + for i, segment := range decoded { + segments[i] = SourceSegment(segment) + } + return segments, nil +} + +type sourceSegmentWithAliases SourceSegment + +// Resolve repeated exact names before field conversion, then apply case +// variants in the same key order as the original object round trip. +func (s *sourceSegmentWithAliases) UnmarshalJSON(raw []byte) error { + if !needsMemberResolution(raw, 1) { + return json.Unmarshal(raw, (*SourceSegment)(s)) + } + var members map[string]json.RawMessage + if err := json.Unmarshal(raw, &members); err != nil { + return err + } + for _, key := range slices.Sorted(maps.Keys(members)) { + var field any + switch sourceMemberFor(key) { + case memberID: + field = &s.ID + case memberSpeaker: + field = &s.Speaker + case memberStart: + field = &s.Start + case memberEnd: + field = &s.End + case memberText: + field = &s.Text + case memberCategories: + field = &s.Categories + default: + continue + } + if err := json.Unmarshal(members[key], field); err != nil { + return fmt.Errorf("field %q: %w", key, err) + } + } + return nil +} + +type sourceMember uint8 + +const ( + memberID sourceMember = 1 << iota + memberSpeaker + memberStart + memberEnd + memberText + memberCategories +) + +func sourceMemberFor(key string) sourceMember { + switch { + case strings.EqualFold(key, "id"): + return memberID + case strings.EqualFold(key, "speaker"): + return memberSpeaker + case strings.EqualFold(key, "start"): + return memberStart + case strings.EqualFold(key, "end"): + return memberEnd + case strings.EqualFold(key, "text"): + return memberText + case strings.EqualFold(key, "categories"): + return memberCategories + } + return 0 +} + +// needsMemberResolution looks for repeated recognized keys in valid JSON. +// segmentDepth locates members within the array or a single segment. +// Escaped keys use the compatibility decoder to resolve their spelling. +func needsMemberResolution(raw []byte, segmentDepth int) bool { + depth := 0 + wantKey := false + var seen sourceMember + for i := 0; i < len(raw); i++ { + switch raw[i] { + case '{': + depth++ + if depth == segmentDepth { + seen, wantKey = 0, true + } + case '[': + depth++ + case '}', ']': + depth-- + case ',': + if depth == segmentDepth { + wantKey = true + } + case '"': + start := i + 1 + escaped := false + for i++; raw[i] != '"'; i++ { + if raw[i] == '\\' { + escaped = true + i++ + } + } + if depth != segmentDepth || !wantKey { + continue + } + wantKey = false + if escaped { + return true + } + member := sourceMemberFor(string(raw[start:i])) + if seen&member != 0 { + return true + } + seen |= member + } + } + return false +} diff --git a/internal/core/schema/transcript.go b/internal/core/schema/transcript.go index 21b2325..6e41400 100644 --- a/internal/core/schema/transcript.go +++ b/internal/core/schema/transcript.go @@ -4,10 +4,8 @@ import ( "bytes" "encoding/json" "fmt" - "maps" "math" "slices" - "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber" ) @@ -21,38 +19,6 @@ type SourceSegment struct { Categories []string `json:"categories,omitempty"` } -// UnmarshalJSON decodes only the final value of each exact key, then applies -// case variants in the same key order as the original object round trip. -func (s *SourceSegment) UnmarshalJSON(raw []byte) error { - var members map[string]json.RawMessage - if err := json.Unmarshal(raw, &members); err != nil { - return err - } - for _, key := range slices.Sorted(maps.Keys(members)) { - var field any - switch { - case strings.EqualFold(key, "id"): - field = &s.ID - case strings.EqualFold(key, "speaker"): - field = &s.Speaker - case strings.EqualFold(key, "start"): - field = &s.Start - case strings.EqualFold(key, "end"): - field = &s.End - case strings.EqualFold(key, "text"): - field = &s.Text - case strings.EqualFold(key, "categories"): - field = &s.Categories - default: - continue - } - if err := json.Unmarshal(members[key], field); err != nil { - return fmt.Errorf("field %q: %w", key, err) - } - } - return nil -} - type Segment struct { ID int `json:"id"` Speaker string `json:"speaker"` @@ -80,8 +46,8 @@ func ParseSourceTranscriptJSON(raw []byte) (*SourceTranscript, error) { return nil, err } - var segments []SourceSegment - if err := json.Unmarshal(segmentsRaw, &segments); err != nil { + segments, err := decodeSourceSegments(segmentsRaw) + if err != nil { return nil, &ParseError{Message: fmt.Sprintf("failed to parse segments: %v", err)} } if len(segments) == 0 { diff --git a/internal/core/schema/transcript_test.go b/internal/core/schema/transcript_test.go index 89b84d1..19d4ff5 100644 --- a/internal/core/schema/transcript_test.go +++ b/internal/core/schema/transcript_test.go @@ -366,6 +366,34 @@ func TestSourceTranscriptUsesEffectiveMemberValues(t *testing.T) { } } +func TestSourceTranscriptCombinesOrdinaryAndAliasedMembers(t *testing.T) { + id := "7" + if strconv.IntSize == 64 { + id = "9007199254740993" + } + wantID, _ := strconv.Atoi(id) + segments := `[ + {"speaker":"A","text":"He said \"{hello, world}\".","id":SOURCE_ID, + "extra":{"speaker":12,"TEXT":"metadata"}}, + {"speaker":17,"spea\u006ber":"B","text":"expected","TEXT":"other","categories":[]}, + {"speaker":"C","ſpeaker":"D","text":"last"} + ]` + segments = strings.Replace(segments, "SOURCE_ID", id, 1) + for _, raw := range []string{segments, `{"segments":` + segments + "}"} { + transcript, err := ParseSourceTranscriptJSON([]byte(raw)) + if err != nil { + t.Fatal(err) + } + got := transcript.Canonical().Segments + if len(got) != 3 || got[0].Speaker != "A" || got[0].ID != wantID || + got[0].Text != `He said "{hello, world}".` || + got[1].Speaker != "B" || got[1].Text != "expected" || got[1].Categories == nil || + got[2].Speaker != "D" || got[2].Text != "last" { + t.Fatalf("unexpected segments: %+v", got) + } + } +} + func TestTranscriptClonePreservesPresenceAndIndependence(t *testing.T) { original := &Transcript{Segments: []Segment{ {ID: 40, Text: "original", Categories: []string{"keep"}},