Collect optional validation context only when it can fit
This commit is contained in:
@@ -14,7 +14,7 @@ type LLMValidationBatch struct {
|
||||
TokenCount int `json:"token_count"`
|
||||
}
|
||||
|
||||
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) {
|
||||
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator, context optionalValidationContext) ([]LLMValidationBatch, []Decision, error) {
|
||||
if estimator == nil {
|
||||
estimator = chunking.NewSimpleTokenEstimator()
|
||||
}
|
||||
@@ -24,7 +24,7 @@ func prepareValidationBatches(items []LLMValidationItem, budget int, estimator c
|
||||
var sizes []int
|
||||
var oversized []Decision
|
||||
for _, item := range items {
|
||||
fitted, err := fitValidationContext(item, budget, estimator)
|
||||
fitted, err := fitValidationContext(item, budget, estimator, context)
|
||||
if err != nil {
|
||||
return nil, oversized, err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package validators
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
)
|
||||
@@ -47,21 +48,31 @@ type fittedValidationItem struct {
|
||||
|
||||
// fitValidationContext tries the required payload followed by glossary entries
|
||||
// and then neighboring segments. The first excess ends the usable prefix.
|
||||
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) {
|
||||
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator, context optionalValidationContext) (result fittedValidationItem, err error) {
|
||||
fitted := fittedValidationItem{item: item}
|
||||
fitted.item.GlossaryEntries = item.GlossaryEntries[:0]
|
||||
fitted.item.NeighboringSegments = item.NeighboringSegments[:0]
|
||||
fitted.item.GlossaryEntries = nil
|
||||
fitted.item.NeighboringSegments = nil
|
||||
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
|
||||
var err error
|
||||
if simple != nil {
|
||||
fitted.cost, err = measureValidationItem(simple, fitted.item)
|
||||
fitted.tokens = fitted.cost.arrayTokens()
|
||||
} else {
|
||||
fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item})
|
||||
}
|
||||
if err != nil || fitted.tokens > budget {
|
||||
if err != nil || fitted.tokens > budget || (simple != nil && fitted.tokens == budget) {
|
||||
return fitted, err
|
||||
}
|
||||
if context != nil {
|
||||
item.NeighboringSegments, item.GlossaryEntries = context(item)
|
||||
}
|
||||
defer func() {
|
||||
if len(result.item.GlossaryEntries) < len(item.GlossaryEntries) {
|
||||
result.item.GlossaryEntries = slices.Clone(result.item.GlossaryEntries)
|
||||
}
|
||||
if len(result.item.NeighboringSegments) < len(item.NeighboringSegments) {
|
||||
result.item.NeighboringSegments = slices.Clone(result.item.NeighboringSegments)
|
||||
}
|
||||
}()
|
||||
try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) {
|
||||
cost := fitted.cost
|
||||
var tokens int
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -97,7 +97,7 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||||
simple := chunking.NewSimpleTokenEstimator()
|
||||
full := serializedTokenCount(t, items)
|
||||
for budget := 1; budget <= full; budget++ {
|
||||
got, rejected, err := prepareValidationBatches(items, budget, simple)
|
||||
got, rejected, err := prepareValidationBatches(items, budget, simple, nil)
|
||||
want, wantRejected := serializedBudgetOracle(t, items, budget)
|
||||
var rejectedIndices []int
|
||||
for _, decision := range rejected {
|
||||
@@ -106,7 +106,9 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||||
}
|
||||
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) {
|
||||
gotJSON, gotErr := json.Marshal(got)
|
||||
wantJSON, wantErr := json.Marshal(want)
|
||||
if err != nil || gotErr != nil || wantErr != nil || !bytes.Equal(gotJSON, wantJSON) || !slices.Equal(rejectedIndices, wantRejected) {
|
||||
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
|
||||
budget, got, rejected, err, want, wantRejected)
|
||||
}
|
||||
@@ -153,7 +155,7 @@ func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
|
||||
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
|
||||
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
|
||||
estimator := &countingEstimator{}
|
||||
fitted, err := fitValidationContext(item, budget, estimator)
|
||||
fitted, err := fitValidationContext(item, budget, estimator, nil)
|
||||
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
|
||||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
|
||||
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
|
||||
@@ -162,8 +164,32 @@ func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
|
||||
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
|
||||
}
|
||||
estimator.measurements = 0
|
||||
fitted, err = fitValidationContext(item, 1, estimator)
|
||||
fitted, err = fitValidationContext(item, 1, estimator, nil)
|
||||
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
|
||||
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationSkipsOptionalContextWithoutBudget(t *testing.T) {
|
||||
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
|
||||
estimator := chunking.NewSimpleTokenEstimator()
|
||||
required := serializedTokenCount(t, []LLMValidationItem{item})
|
||||
loads := 0
|
||||
optional := func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
|
||||
loads++
|
||||
return nil, []glossary.Entry{{Name: "optional"}}
|
||||
}
|
||||
for _, budget := range []int{required - 1, required} {
|
||||
batches, oversized, err := prepareValidationBatches([]LLMValidationItem{item}, budget, estimator, optional)
|
||||
if err != nil || loads != 0 {
|
||||
t.Fatalf("budget=%d loads=%d error=%v", budget, loads, err)
|
||||
}
|
||||
if budget < required {
|
||||
if len(batches) != 0 || len(oversized) != 1 {
|
||||
t.Fatalf("oversized result: batches=%+v rejected=%+v", batches, oversized)
|
||||
}
|
||||
} else if len(batches) != 1 || len(oversized) != 0 || batches[0].TokenCount != required {
|
||||
t.Fatalf("exact fit: batches=%+v rejected=%+v", batches, oversized)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,34 +33,58 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
|
||||
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
|
||||
}
|
||||
|
||||
// validationContext supplies adjacent source segments within the current chunk
|
||||
// and glossary definitions mentioned in the candidate or that local context.
|
||||
// These fields travel in each item so batching accounts for their size.
|
||||
func (view glossaryContext) validationContext(segments []schema.Segment, position int, corrected string) ([]NeighboringSegment, []glossary.Entry) {
|
||||
var neighbors []NeighboringSegment
|
||||
source := segments[position].Text
|
||||
if position > 0 {
|
||||
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
|
||||
}
|
||||
if position+1 < len(segments) {
|
||||
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
|
||||
}
|
||||
type optionalValidationContext func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry)
|
||||
|
||||
texts := []string{source, corrected}
|
||||
for _, neighbor := range neighbors {
|
||||
texts = append(texts, neighbor.Text)
|
||||
// validationContextFor builds optional context on demand. Unchanged segment
|
||||
// matches are shared within the request. Corrected text is candidate-specific.
|
||||
func validationContextFor(req Request) optionalValidationContext {
|
||||
segments := req.sectionLookup().segments
|
||||
var view *glossaryContext
|
||||
if req.GlossaryView != nil {
|
||||
view = &req.GlossaryView.context
|
||||
}
|
||||
mentioned := view.vocabulary.MentionedIdentities(texts...)
|
||||
var matched []int
|
||||
for identity := range mentioned {
|
||||
matched = append(matched, view.positions[identity]...)
|
||||
var sourceMentions map[int]map[glossary.Identity]struct{}
|
||||
return func(item LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
|
||||
position := item.segmentPosition
|
||||
var neighbors []NeighboringSegment
|
||||
if position > 0 {
|
||||
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
|
||||
}
|
||||
if position+1 < len(segments) {
|
||||
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
|
||||
}
|
||||
if view == nil {
|
||||
prepared := prepareGlossaryContext(req.Glossary)
|
||||
view = &prepared
|
||||
}
|
||||
if view.vocabulary.Empty() {
|
||||
return neighbors, nil
|
||||
}
|
||||
if sourceMentions == nil {
|
||||
sourceMentions = make(map[int]map[glossary.Identity]struct{})
|
||||
}
|
||||
mentioned := view.vocabulary.MentionedIdentities(item.CorrectedSegmentText)
|
||||
for i := max(0, position-1); i < min(len(segments), position+2); i++ {
|
||||
identities, ok := sourceMentions[i]
|
||||
if !ok {
|
||||
identities = view.vocabulary.MentionedIdentities(segments[i].Text)
|
||||
sourceMentions[i] = identities
|
||||
}
|
||||
for id := range identities {
|
||||
mentioned[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
var matched []int
|
||||
for identity := range mentioned {
|
||||
matched = append(matched, view.positions[identity]...)
|
||||
}
|
||||
slices.Sort(matched)
|
||||
var entries []glossary.Entry
|
||||
for _, index := range matched {
|
||||
entries = append(entries, view.entries[index])
|
||||
}
|
||||
return neighbors, entries
|
||||
}
|
||||
slices.Sort(matched)
|
||||
var entries []glossary.Entry
|
||||
for _, index := range matched {
|
||||
entries = append(entries, view.entries[index])
|
||||
}
|
||||
return neighbors, entries
|
||||
}
|
||||
|
||||
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
|
||||
|
||||
@@ -11,6 +11,16 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
|
||||
func buildContextItems(req Request) ([]LLMValidationItem, []Decision) {
|
||||
req.Section = req.sectionLookup()
|
||||
items, immediate := buildValidationItems(req)
|
||||
context := validationContextFor(req)
|
||||
for i := range items {
|
||||
items[i].NeighboringSegments, items[i].GlossaryEntries = context(items[i])
|
||||
}
|
||||
return items, immediate
|
||||
}
|
||||
|
||||
func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters")})
|
||||
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{
|
||||
@@ -25,7 +35,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
{ID: "lyra", Name: "Lyra", Description: "The host"},
|
||||
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
|
||||
}
|
||||
payload, immediate := buildValidationItems(req)
|
||||
payload, immediate := buildContextItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 1 {
|
||||
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
|
||||
}
|
||||
@@ -51,7 +61,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
|
||||
req.WorkingTranscript.Segments[0].Speaker = ""
|
||||
req.Glossary = nil
|
||||
isolated, _ := buildValidationItems(req)
|
||||
isolated, _ := buildContextItems(req)
|
||||
if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 {
|
||||
t.Fatalf("unexpected external context: %+v", isolated)
|
||||
}
|
||||
@@ -73,7 +83,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
|
||||
enriched := base
|
||||
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
|
||||
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
|
||||
fit, err := fitValidationContext(enriched, required, estimator)
|
||||
fit, err := fitValidationContext(enriched, required, estimator, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -91,15 +101,17 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
|
||||
|
||||
func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
|
||||
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
|
||||
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
|
||||
payload, immediate := buildValidationItems(req)
|
||||
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Text: "Those gestures are back."}, {ID: 2, Text: "They met Lyra."},
|
||||
}}
|
||||
req.Glossary = []glossary.Entry{{ID: "lyra", Name: "Lyra"}, {ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
|
||||
payload, immediate := buildContextItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 2 {
|
||||
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
|
||||
}
|
||||
for i, name := range []string{"Jesters", "Actors"} {
|
||||
entries := payload[i].GlossaryEntries
|
||||
if len(entries) != 1 || entries[0].Name != name {
|
||||
if len(entries) != 2 || entries[0].Name != "Lyra" || entries[1].Name != name {
|
||||
t.Fatalf("candidate %d context: %+v", i, entries)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ type StructuredLLMClient = structuredoutput.Client
|
||||
// LLMValidationItem retains the internal proposal index until prompt encoding.
|
||||
// BuildValidationVariables substitutes a one-based reference for each batch.
|
||||
type LLMValidationItem struct {
|
||||
segmentPosition int
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Speaker string `json:"speaker"`
|
||||
OriginalText string `json:"original_text"`
|
||||
|
||||
@@ -60,6 +60,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
|
||||
}
|
||||
req.Section = req.sectionLookup()
|
||||
items, immediate := buildValidationItems(req)
|
||||
if len(items) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
@@ -75,7 +76,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
maxTokens = req.Settings.Validation.MaxPromptTokens
|
||||
}
|
||||
|
||||
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator)
|
||||
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator, validationContextFor(req))
|
||||
operations := make([]outcomes.Operation, 0)
|
||||
llmDecisions := make([]Decision, 0)
|
||||
partialResult := func() Result {
|
||||
@@ -253,13 +254,6 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
|
||||
immediate := make([]Decision, 0)
|
||||
req.Section = req.sectionLookup()
|
||||
segments := req.Section.segments
|
||||
var glossary glossaryContext
|
||||
if req.GlossaryView != nil {
|
||||
glossary = req.GlossaryView.context
|
||||
} else {
|
||||
glossary = prepareGlossaryContext(req.Glossary)
|
||||
}
|
||||
|
||||
for _, p := range req.CandidateProposal {
|
||||
position, ok := req.Section.Position(p.TargetSegmentID)
|
||||
if !ok {
|
||||
@@ -274,8 +268,8 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
|
||||
continue
|
||||
}
|
||||
|
||||
neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText)
|
||||
items = append(items, LLMValidationItem{
|
||||
segmentPosition: position,
|
||||
CorrectionIndex: p.ProposalIndex,
|
||||
Speaker: seg.Speaker,
|
||||
OriginalText: p.OriginalText,
|
||||
@@ -283,8 +277,6 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
|
||||
OriginalSegmentText: seg.Text,
|
||||
CorrectedSegmentText: preview.CorrectedSegmentText,
|
||||
Categories: append([]string(nil), seg.Categories...),
|
||||
NeighboringSegments: neighbors,
|
||||
GlossaryEntries: glossaryEntries,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -133,7 +133,8 @@ func TestValidationContextIncludesEachSharedDefinition(t *testing.T) {
|
||||
{Collection: "items", ID: "bow", Name: "Bow", Category: "item"},
|
||||
}
|
||||
segments := []schema.Segment{{ID: 1, Text: "Shield Shield"}}
|
||||
_, selected := prepareGlossaryContext(entries).validationContext(segments, 0, "Sword")
|
||||
context := validationContextFor(Request{WorkingTranscript: &schema.Transcript{Segments: segments}, Glossary: entries})
|
||||
_, selected := context(LLMValidationItem{CorrectedSegmentText: "Sword"})
|
||||
if len(selected) != 3 || selected[0].ID != "sword" || selected[1].Category != "spell" || selected[2].ID != "shield" || selected[2].Collection != "items" {
|
||||
t.Fatalf("selected=%+v", selected)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user