Keep transcript decoding and encoding on typed data

This commit is contained in:
2026-11-16 03:56:54 +00:00
parent b63c8af467
commit f465496dd8
19 changed files with 286 additions and 386 deletions

View File

@@ -378,11 +378,11 @@ func (f processFixture) readTranscript(t *testing.T, path string) *schema.Transc
if err != nil {
t.Fatal(err)
}
transcript, err := schema.ParseTranscriptJSON(raw)
transcript, err := schema.ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatal(err)
}
return transcript
return transcript.Canonical()
}
func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) {

View File

@@ -13,7 +13,7 @@ schema/version marker.
| Segment field | Type and validation |
| --- | --- |
| `id` | Optional integer (or null). Supplied IDs must be unique; missing IDs are accepted. |
| `id` | Optional integer (or null) in the Go `int` range for the executable's architecture. Supplied IDs are compared exactly and must be unique. |
| `speaker` | Non-empty string |
| `start`, `end` | Numeric seconds, finite and non-negative; end must be at least start |
| `text` | Non-empty string |
@@ -21,8 +21,8 @@ schema/version marker.
Supply explicit start and end times for meaningful timing. The current decoder
uses zero for missing or null timestamps rather than enforcing their presence.
String validation checks empty values, not whitespace trimming. Normalization can
merge segments and reassign IDs, so source IDs need not match output IDs.
String validation checks for empty values. Normalization can merge segments and
reassign IDs, so source IDs need not match output IDs.
The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyable
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field

View File

@@ -17,7 +17,9 @@ An unsupported key fails validation or resolution rather than selecting an
implicit fallback.
The registry delegates bare segment serialization to the transcript schema
package and uses an envelope encoder for the versioned form. Report construction
package and uses an envelope encoder for the versioned form. Both serialize
the model's typed segment array directly, including native integer IDs and
omitting empty categories. Report construction
receives the selected key separately; see [report assembly](diagnostics-reporting.md).
## Changing an Encoder

View File

@@ -221,6 +221,13 @@ and the distinction between raw and typed debug snapshots. Update the
[input contract](../integrations/transcript-glossary-files.md) and
[encoder internals](../internal/output-schemas.md) where affected.
**Complete.** Source segments decode directly from both accepted JSON shapes.
Exact IDs, clone independence, encoder values, and debug snapshot fidelity were
verified. Processing releases raw bytes after parsing or source capture and
releases the parsed view before normalization. On a 100,000-segment object
fixture (Go 1.27.3, five iterations), parser allocations fell from 240.6 MB to
84.1 MB per operation. The CLI build and full offline suite passed.
## Stage 6 — Correct and simplify normalization
Audit #17, #21, #35, #53, #59, #62 (merge helper).

View File

@@ -80,7 +80,7 @@ func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
if _, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}); err != nil {
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}); err != nil {
t.Fatal(err)
}
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {

View File

@@ -52,12 +52,15 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
if debug != nil {
warn(debug.WriteSourceTranscript(loaded.Source, loaded.RawTranscript))
}
loaded.RawTranscript = nil
cfg := &loaded.Configuration.Config
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
MaxSegmentGap: cfg.Normalization.MaxSegmentGap, EllipsisGap: cfg.Normalization.EllipsisGap,
MaxSegmentDuration: cfg.Normalization.MaxSegmentDuration, MaxSegmentTokens: cfg.Normalization.MaxSegmentTokens,
})
normalized, summary := normalizer.Normalize(sourceToCanonicalTranscript(loaded.Source))
transcript := loaded.Source.Canonical()
loaded.Source = nil
normalized, summary := normalizer.Normalize(transcript)
if debug != nil {
warn(debug.WriteNormalizedTranscript(normalized))
warn(debug.WriteNormalizationSummary(summary))

View File

@@ -58,6 +58,9 @@ func loadProcessInputs(ctx context.Context, input processInput, mode commandkit.
if err != nil {
return nil, err
}
if !input.Debug.Value {
raw = nil
}
protected := append([]string(nil), runtime.SourcePaths...)
for _, entry := range loaded.Sources() {
if entry.Kind == ck.SourceKindFile && entry.Path != "" {

View File

@@ -122,7 +122,7 @@ pipeline:
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
if err != nil {
t.Fatal(err)
}
@@ -194,7 +194,7 @@ func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {})
if err != nil {
t.Fatal(err)
}

View File

@@ -156,7 +156,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(inv.Context, loaded, prepared, sourceToCanonicalTranscript(loaded.Source), func(error) {})
result, err := executePipeline(inv.Context, loaded, prepared, loaded.Source.Canonical(), func(error) {})
if err != nil {
t.Fatal(err)
}

View File

@@ -5,29 +5,9 @@ import (
"fmt"
"io"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
commandkit "go.maximumdirect.net/commandkit"
)
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
segments := make([]schema.Segment, len(source.Segments))
for i, s := range source.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
return &schema.Transcript{Segments: segments}
}
// Run executes the application with the provided arguments and streams.
func Run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
app, err := buildApplication()

View File

@@ -1,7 +1,6 @@
package io
import (
"encoding/json"
"fmt"
"os"
)
@@ -13,17 +12,3 @@ func ReadRequiredFile(path string, label string) ([]byte, error) {
}
return contents, nil
}
func ValidateWellFormedJSON(path string, raw []byte) error {
if !json.Valid(raw) {
return fmt.Errorf("transcript file %q is not valid JSON", path)
}
return nil
}
func WriteFile(path string, contents []byte) error {
if err := os.WriteFile(path, contents, 0o644); err != nil {
return fmt.Errorf("failed to write output file %q: %w", path, err)
}
return nil
}

View File

@@ -1,6 +1,7 @@
package normalization
import (
"encoding/json"
"os"
"strings"
"testing"
@@ -292,25 +293,7 @@ func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
t.Fatalf("failed to parse source transcript: %v", err)
}
converted := &schema.Transcript{
Segments: make([]schema.Segment, len(transcript.Segments)),
}
for i, s := range transcript.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
converted.Segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
normalized, _ := normalizer.Normalize(converted)
normalized, _ := normalizer.Normalize(transcript.Canonical())
// Compare with golden output semantically
expectedRaw, err := os.ReadFile("testdata/bare_array_transcript.golden.json")
@@ -318,12 +301,12 @@ func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
var expectedTranscript schema.Transcript
if err := json.Unmarshal(expectedRaw, &expectedTranscript.Segments); err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, expectedTranscript)
assertTranscriptsEqual(t, normalized, &expectedTranscript)
}
func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
@@ -345,25 +328,7 @@ func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
t.Fatalf("failed to parse source transcript: %v", err)
}
converted := &schema.Transcript{
Segments: make([]schema.Segment, len(transcript.Segments)),
}
for i, s := range transcript.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
converted.Segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
normalized, _ := normalizer.Normalize(converted)
normalized, _ := normalizer.Normalize(transcript.Canonical())
// Compare with golden output semantically
expectedRaw, err := os.ReadFile("testdata/object_with_segments_transcript.golden.json")
@@ -371,12 +336,12 @@ func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
var expectedTranscript schema.Transcript
if err := json.Unmarshal(expectedRaw, &expectedTranscript.Segments); err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, expectedTranscript)
assertTranscriptsEqual(t, normalized, &expectedTranscript)
}
func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
@@ -398,25 +363,7 @@ func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
t.Fatalf("failed to parse source transcript: %v", err)
}
converted := &schema.Transcript{
Segments: make([]schema.Segment, len(transcript.Segments)),
}
for i, s := range transcript.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
converted.Segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
normalized, _ := normalizer.Normalize(converted)
normalized, _ := normalizer.Normalize(transcript.Canonical())
// Compare with golden output semantically
expectedRaw, err := os.ReadFile("testdata/transcript_with_categories.golden.json")
@@ -424,12 +371,12 @@ func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
var expectedTranscript schema.Transcript
if err := json.Unmarshal(expectedRaw, &expectedTranscript.Segments); err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, expectedTranscript)
assertTranscriptsEqual(t, normalized, &expectedTranscript)
}
func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
@@ -451,25 +398,7 @@ func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
t.Fatalf("failed to parse source transcript: %v", err)
}
converted := &schema.Transcript{
Segments: make([]schema.Segment, len(transcript.Segments)),
}
for i, s := range transcript.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
converted.Segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
normalized, _ := normalizer.Normalize(converted)
normalized, _ := normalizer.Normalize(transcript.Canonical())
// Compare with golden output semantically
expectedRaw, err := os.ReadFile("testdata/transcript_with_original_ids.golden.json")
@@ -477,12 +406,12 @@ func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
var expectedTranscript schema.Transcript
if err := json.Unmarshal(expectedRaw, &expectedTranscript.Segments); err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, expectedTranscript)
assertTranscriptsEqual(t, normalized, &expectedTranscript)
}
func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
@@ -504,25 +433,7 @@ func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
t.Fatalf("failed to parse source transcript: %v", err)
}
converted := &schema.Transcript{
Segments: make([]schema.Segment, len(transcript.Segments)),
}
for i, s := range transcript.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
converted.Segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
normalized, _ := normalizer.Normalize(converted)
normalized, _ := normalizer.Normalize(transcript.Canonical())
// Compare with golden output semantically
expectedRaw, err := os.ReadFile("testdata/transcript_same_speaker_merge.golden.json")
@@ -530,12 +441,12 @@ func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
var expectedTranscript schema.Transcript
if err := json.Unmarshal(expectedRaw, &expectedTranscript.Segments); err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, expectedTranscript)
assertTranscriptsEqual(t, normalized, &expectedTranscript)
}
func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
@@ -557,25 +468,7 @@ func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
t.Fatalf("failed to parse source transcript: %v", err)
}
converted := &schema.Transcript{
Segments: make([]schema.Segment, len(transcript.Segments)),
}
for i, s := range transcript.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
converted.Segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
normalized, _ := normalizer.Normalize(converted)
normalized, _ := normalizer.Normalize(transcript.Canonical())
// Compare with golden output semantically
expectedRaw, err := os.ReadFile("testdata/transcript_ellipsis_merge.golden.json")
@@ -583,12 +476,12 @@ func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
var expectedTranscript schema.Transcript
if err := json.Unmarshal(expectedRaw, &expectedTranscript.Segments); err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, expectedTranscript)
assertTranscriptsEqual(t, normalized, &expectedTranscript)
}
func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
@@ -610,25 +503,7 @@ func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
t.Fatalf("failed to parse source transcript: %v", err)
}
converted := &schema.Transcript{
Segments: make([]schema.Segment, len(transcript.Segments)),
}
for i, s := range transcript.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
converted.Segments[i] = schema.Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
normalized, _ := normalizer.Normalize(converted)
normalized, _ := normalizer.Normalize(transcript.Canonical())
// Compare with golden output semantically
expectedRaw, err := os.ReadFile("testdata/transcript_different_speakers_no_merge.golden.json")
@@ -636,12 +511,12 @@ func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
var expectedTranscript schema.Transcript
if err := json.Unmarshal(expectedRaw, &expectedTranscript.Segments); err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, expectedTranscript)
assertTranscriptsEqual(t, normalized, &expectedTranscript)
}
func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {

View File

@@ -60,29 +60,12 @@ func Resolve(key string) (Definition, error) {
}
func encodeAuditaV1(transcript *schema.Transcript) ([]byte, error) {
if transcript == nil {
transcript = &schema.Transcript{}
}
payload := map[string]any{
"schema": "audita-v1",
"version": "v1",
"segments": func() []map[string]any {
out := make([]map[string]any, len(transcript.Segments))
for i, s := range transcript.Segments {
item := map[string]any{
"id": s.ID,
"speaker": s.Speaker,
"start": s.Start,
"end": s.End,
"text": s.Text,
}
if len(s.Categories) > 0 {
item["categories"] = s.Categories
}
out[i] = item
}
return out
}(),
payload := struct {
Schema string `json:"schema"`
Version string `json:"version"`
Segments []schema.Segment `json:"segments"`
}{
Schema: SchemaAuditaV1, Version: "v1", Segments: transcript.OutputSegments(),
}
return json.MarshalIndent(payload, "", " ")
}

View File

@@ -2,7 +2,9 @@ package outputschema
import (
"encoding/json"
"math"
"reflect"
"strconv"
"strings"
"testing"
@@ -73,3 +75,41 @@ func TestSupportedKeysAndIsSupported(t *testing.T) {
t.Fatalf("did not expect unsupported schema to be reported as supported")
}
}
func TestEncodersPreserveSegmentValues(t *testing.T) {
id := 5
if strconv.IntSize == 64 {
id, _ = strconv.Atoi("9007199254740993")
}
input := &schema.Transcript{Segments: []schema.Segment{
{ID: id, Speaker: "A", Start: 0.25, End: 2.75, Text: "first", Categories: []string{"one", "two"}},
{ID: 6, Speaker: "B", Start: 3, End: 5, Text: "second", Categories: []string{}},
}}
for _, key := range []string{SchemaBareSegments, SchemaAuditaV1} {
t.Run(key, func(t *testing.T) {
definition, err := Resolve(key)
if err != nil {
t.Fatal(err)
}
payload, err := definition.Encoder(input)
if err != nil {
t.Fatal(err)
}
decoded, err := schema.ParseSourceTranscriptJSON(payload)
if err != nil {
t.Fatal(err)
}
want := []schema.Segment{
{ID: id, Speaker: "A", Start: 0.25, End: 2.75, Text: "first", Categories: []string{"one", "two"}},
{ID: 6, Speaker: "B", Start: 3, End: 5, Text: "second"},
}
if got := decoded.Canonical().Segments; !reflect.DeepEqual(got, want) {
t.Fatalf("decoded %s: %+v", key, got)
}
invalid := &schema.Transcript{Segments: []schema.Segment{{Start: math.NaN()}}}
if _, err := definition.Encoder(invalid); err == nil {
t.Fatal("nonfinite timestamp encoded successfully")
}
})
}
}

View File

@@ -1,18 +1,22 @@
package schema
import (
"bytes"
"encoding/json"
"fmt"
"math"
"slices"
"strconv"
"strings"
)
type SourceSegment struct {
ID *int `json:"id,omitempty"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
ID *sourceID `json:"id,omitempty"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type Segment struct {
@@ -37,12 +41,7 @@ func ParseSourceTranscriptJSON(raw []byte) (*SourceTranscript, error) {
return nil, &ParseError{Message: "transcript is not valid JSON"}
}
var top any
if err := json.Unmarshal(raw, &top); err != nil {
return nil, &ParseError{Message: fmt.Sprintf("failed to parse transcript JSON: %v", err)}
}
segmentsRaw, err := extractSegments(top)
segmentsRaw, err := extractSegments(bytes.TrimSpace(raw))
if err != nil {
return nil, err
}
@@ -62,83 +61,113 @@ func ParseSourceTranscriptJSON(raw []byte) (*SourceTranscript, error) {
return &SourceTranscript{Segments: segments}, nil
}
func ParseTranscriptJSON(raw []byte) (*Transcript, error) {
source, err := ParseSourceTranscriptJSON(raw)
if err != nil {
return nil, err
}
segments := make([]Segment, len(source.Segments))
for i, s := range source.Segments {
segments[i] = Segment{
ID: *s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
}
}
if err := validateSequentialIDs(segments); err != nil {
return nil, err
}
return &Transcript{Segments: segments}, nil
}
func ParseTranscriptJSONLenient(raw []byte) (*Transcript, error) {
source, err := ParseSourceTranscriptJSON(raw)
if err != nil {
return nil, err
}
segments := make([]Segment, len(source.Segments))
for i, s := range source.Segments {
// Canonical gives segments without source IDs their one-based position.
func (t *SourceTranscript) Canonical() *Transcript {
segments := make([]Segment, len(t.Segments))
for i, s := range t.Segments {
id := i + 1
if s.ID != nil {
id = *s.ID
}
segments[i] = Segment{
ID: id,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: s.Categories,
id = int(*s.ID)
}
segments[i] = Segment{ID: id, Speaker: s.Speaker, Start: s.Start, End: s.End, Text: s.Text, Categories: slices.Clone(s.Categories)}
}
return &Transcript{Segments: segments}, nil
return &Transcript{Segments: segments}
}
func extractSegments(top any) (json.RawMessage, error) {
switch v := top.(type) {
case []any:
raw, err := json.Marshal(v)
if err != nil {
return nil, &ParseError{Message: "failed to re-encode segment array"}
}
// Clone copies the segment and category slices for independent mutation.
func (t *Transcript) Clone() *Transcript {
if t == nil {
return &Transcript{}
}
segments := slices.Clone(t.Segments)
for i := range segments {
segments[i].Categories = slices.Clone(segments[i].Categories)
}
return &Transcript{Segments: segments}
}
func extractSegments(raw []byte) (json.RawMessage, error) {
switch raw[0] {
case '[':
return raw, nil
case map[string]any:
segs, ok := v["segments"]
case '{':
var envelope map[string]json.RawMessage
if err := json.Unmarshal(raw, &envelope); err != nil {
return nil, &ParseError{Message: fmt.Sprintf("failed to parse transcript JSON: %v", err)}
}
segments, ok := envelope["segments"]
if !ok {
return nil, &ParseError{Message: "transcript object must contain a segments array"}
}
segsArray, ok := segs.([]any)
if !ok {
segments = bytes.TrimSpace(segments)
if len(segments) == 0 || segments[0] != '[' {
return nil, &ParseError{Message: "segments field must be an array"}
}
raw, err := json.Marshal(segsArray)
if err != nil {
return nil, &ParseError{Message: "failed to re-encode segments array"}
}
return raw, nil
return segments, nil
default:
return nil, &ParseError{Message: "transcript must be a JSON array or an object with a segments array"}
}
}
// sourceID decodes native integers without routing them through float64.
// Decimal and exponent spellings are accepted if they denote an exact integer.
type sourceID int
func (id *sourceID) UnmarshalJSON(raw []byte) error {
spelling := string(raw)
if value, err := strconv.Atoi(spelling); err == nil {
*id = sourceID(value)
return nil
}
if value, ok := integralDecimal(spelling); ok {
*id = sourceID(value)
return nil
}
return json.Unmarshal(raw, (*int)(id))
}
func integralDecimal(value string) (int, bool) {
if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") {
return 0, false
}
digits, negative := strings.CutPrefix(value, "-")
var exponentText string
if i := strings.IndexAny(digits, "eE"); i >= 0 {
exponentText, digits = digits[i+1:], digits[:i]
}
whole, fraction, _ := strings.Cut(digits, ".")
digits = strings.TrimLeft(whole+fraction, "0")
if digits == "" {
return 0, true
}
var exponent int
if exponentText != "" {
var err error
exponent, err = strconv.Atoi(exponentText)
if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 {
return 0, false
}
}
shift := exponent - len(fraction)
if shift < 0 {
// A negative shift is integral only if every discarded digit is zero.
if -shift > len(digits)-len(strings.TrimRight(digits, "0")) {
return 0, false
}
digits = digits[:len(digits)+shift]
} else {
if len(digits)+shift > 19 {
return 0, false
}
digits += strings.Repeat("0", shift)
}
if negative {
digits = "-" + digits
}
integer, err := strconv.Atoi(digits)
return integer, err == nil
}
func validateSourceSegments(segments []SourceSegment) error {
for i, s := range segments {
segLabel := segmentLabel(i, s.ID)
@@ -178,13 +207,13 @@ func validateSourceSegments(segments []SourceSegment) error {
seenIDs := make(map[int]int)
for i, s := range segments {
if s.ID != nil {
if firstIdx, exists := seenIDs[*s.ID]; exists {
if firstIdx, exists := seenIDs[int(*s.ID)]; exists {
return &ValidationError{
Field: fmt.Sprintf("segment[%d].id", i),
Message: fmt.Sprintf("duplicate id %d (first used at segment[%d])", *s.ID, firstIdx),
}
}
seenIDs[*s.ID] = i
seenIDs[int(*s.ID)] = i
}
}
@@ -201,39 +230,22 @@ func validateTime(t float64, field string) error {
return nil
}
func validateSequentialIDs(segments []Segment) error {
for i, s := range segments {
expected := i + 1
if s.ID != expected {
return &ValidationError{
Field: fmt.Sprintf("segment[%d].id", i),
Message: fmt.Sprintf("must be sequential starting at 1 (got %d, expected %d)", s.ID, expected),
}
}
}
return nil
}
func segmentLabel(index int, id *int) string {
func segmentLabel(index int, id *sourceID) string {
if id != nil {
return fmt.Sprintf("segment[%d] (id=%d)", index, *id)
}
return fmt.Sprintf("segment[%d]", index)
}
func TranscriptToJSON(t *Transcript) ([]byte, error) {
payload := make([]map[string]any, len(t.Segments))
for i, s := range t.Segments {
payload[i] = map[string]any{
"id": s.ID,
"speaker": s.Speaker,
"start": s.Start,
"end": s.End,
"text": s.Text,
}
if len(s.Categories) > 0 {
payload[i]["categories"] = s.Categories
}
// OutputSegments supplies the typed segment array used by both output envelopes.
// An empty transcript is encoded as an empty JSON array.
func (t *Transcript) OutputSegments() []Segment {
if t == nil || t.Segments == nil {
return []Segment{}
}
return json.MarshalIndent(payload, "", " ")
return t.Segments
}
func TranscriptToJSON(t *Transcript) ([]byte, error) {
return json.MarshalIndent(t.OutputSegments(), "", " ")
}

View File

@@ -2,6 +2,8 @@ package schema
import (
"os"
"strconv"
"strings"
"testing"
)
@@ -163,30 +165,33 @@ func TestParseSourceTranscriptJSON_EmptyArray(t *testing.T) {
}
}
func TestParseTranscriptJSON_NonSequentialID(t *testing.T) {
func TestCanonicalPreservesNonSequentialSourceIDs(t *testing.T) {
raw, err := os.ReadFile("testdata/transcript_non_sequential_id.json")
if err != nil {
t.Fatalf("failed to read test fixture: %v", err)
}
_, err = ParseTranscriptJSON(raw)
if err == nil {
t.Fatal("expected error for non-sequential ID, got nil")
source, err := ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatal(err)
}
canonical := source.Canonical()
if len(canonical.Segments) != 1 || canonical.Segments[0].ID != 2 {
t.Fatalf("IDs: %+v", canonical.Segments)
}
assertValidationErrorContains(t, err, "id", "must be sequential")
}
func TestParseTranscriptJSONLenient_AssignsSequentialIDs(t *testing.T) {
func TestCanonicalAssignsMissingIDs(t *testing.T) {
raw, err := os.ReadFile("testdata/transcript_no_ids.json")
if err != nil {
t.Fatalf("failed to read test fixture: %v", err)
}
transcript, err := ParseTranscriptJSONLenient(raw)
source, err := ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatalf("ParseTranscriptJSONLenient failed: %v", err)
t.Fatal(err)
}
transcript := source.Canonical()
if len(transcript.Segments) != 2 {
t.Fatalf("expected 2 segments, got %d", len(transcript.Segments))
@@ -245,7 +250,7 @@ func TestTranscriptToJSON(t *testing.T) {
t.Fatalf("TranscriptToJSON failed: %v", err)
}
parsed, err := ParseTranscriptJSON(raw)
parsed, err := ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatalf("failed to parse round-trip JSON: %v", err)
}
@@ -295,3 +300,56 @@ func assertValidationErrorContains(t *testing.T, err error, fieldContains, messa
t.Errorf("expected message to contain %q, got empty message", messageContains)
}
}
func TestSourceIDsAreDecodedExactly(t *testing.T) {
first, second := "1", "2"
if strconv.IntSize == 64 {
first, second = "9007199254740992", "9007199254740993"
}
array := `[{"id":FIRST,"speaker":"A","text":"one","ignored":1e1000},{"id":SECONDe0,"speaker":"B","text":"two"},{"id":null,"speaker":"C","text":"three","categories":[]}]`
array = strings.NewReplacer("FIRST", first, "SECOND", second).Replace(array)
wantFirst, _ := strconv.Atoi(first)
wantSecond, _ := strconv.Atoi(second)
for _, raw := range []string{array, `{"segments":` + array + `,"ignored":1e1000}`} {
source, err := ParseSourceTranscriptJSON([]byte(raw))
if err != nil {
t.Fatal(err)
}
segments := source.Canonical().Segments
if len(segments) != 3 || segments[0].ID != wantFirst || segments[1].ID != wantSecond || segments[2].ID != 3 || source.Segments[2].ID != nil {
t.Fatalf("source IDs and canonical segments: %+v, %+v", source.Segments, segments)
}
if segments[0].Categories != nil || segments[2].Categories == nil || len(segments[2].Categories) != 0 {
t.Fatalf("category presence: %+v", segments)
}
}
}
func TestSourceIDRejectsFractionAndOverflow(t *testing.T) {
for _, id := range []string{`"1"`, "0.1", "1e1000", "1e-1000", "9223372036854775808"} {
t.Run(id, func(t *testing.T) {
_, err := ParseSourceTranscriptJSON([]byte(`[{"speaker":"A","text":"one","id":` + id + `}]`))
if _, ok := err.(*ParseError); !ok {
t.Fatalf("expected a parse error for %s, got %v", id, err)
}
})
}
}
func TestTranscriptClonePreservesPresenceAndIndependence(t *testing.T) {
original := &Transcript{Segments: []Segment{
{ID: 40, Text: "original", Categories: []string{"keep"}},
{ID: 41, Categories: []string{}},
{ID: 42},
}}
clone := original.Clone()
clone.Segments[0].Text = "changed"
clone.Segments[0].Categories[0] = "changed"
if original.Segments[0].Text != "original" || original.Segments[0].Categories[0] != "keep" {
t.Fatalf("clone mutated original: %+v", original)
}
if clone.Segments[1].Categories == nil || clone.Segments[2].Categories != nil ||
(&Transcript{}).Clone().Segments != nil || (&Transcript{Segments: []Segment{}}).Clone().Segments == nil {
t.Fatalf("clone changed slice presence: %+v", clone)
}
}

View File

@@ -45,7 +45,7 @@ func ApplyProposals(
policy ReplacementPolicy,
allowsChange func(before, after string) bool,
) ApplyResult {
working := cloneTranscript(transcript)
working := transcript.Clone()
ordered := sortProposalsByIndex(proposals)
applied := make([]AppliedChange, 0, len(ordered))
@@ -104,32 +104,6 @@ func sortProposalsByIndex(proposals []EnrichedCorrectionProposal) []EnrichedCorr
return ordered
}
func cloneTranscript(t *schema.Transcript) *schema.Transcript {
if t == nil {
return &schema.Transcript{}
}
segments := make([]schema.Segment, len(t.Segments))
for i, s := range t.Segments {
var categories []string
if s.Categories != nil {
categories = make([]string, len(s.Categories))
copy(categories, s.Categories)
}
segments[i] = schema.Segment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: categories,
}
}
return &schema.Transcript{Segments: segments}
}
func newSkippedChange(proposal EnrichedCorrectionProposal, reason ProposalSkipReason) SkippedChange {
return SkippedChange{
ProposalIndex: proposal.ProposalIndex,

View File

@@ -198,7 +198,7 @@ func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) {
func TestApplyProposalsNoMutationOfInputTranscript(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}}
before := cloneTranscript(transcript)
before := transcript.Clone()
proposals := []EnrichedCorrectionProposal{
{

View File

@@ -118,7 +118,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
return RunOutput{}, fmt.Errorf("runner module factory is required")
}
working := cloneTranscript(input.Transcript)
working := input.Transcript.Clone()
results := make([]ModuleResult, 0, len(input.ModuleSpecs))
sections, err := chunkWorkingTranscript(input.Config, working)
if err != nil {
@@ -939,25 +939,3 @@ func transcriptFromSection(section chunking.Section) *schema.Transcript {
copy(segments, section.Segments)
return &schema.Transcript{Segments: segments}
}
func cloneTranscript(t *schema.Transcript) *schema.Transcript {
if t == nil {
return &schema.Transcript{}
}
segments := make([]schema.Segment, len(t.Segments))
for i, s := range t.Segments {
var categories []string
if s.Categories != nil {
categories = append([]string(nil), s.Categories...)
}
segments[i] = schema.Segment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: categories,
}
}
return &schema.Transcript{Segments: segments}
}