package validators import ( "encoding/json" "fmt" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" ) // Compact JSON fragments in this budget begin and end with non-whitespace. // When fragments join, their touching word runs merge. Counting only extra // runs inside each fragment makes costs additive, with one run for the whole // payload and punctuation rounded only after assembling the array. type validationCost struct { extraRuns int punctuation int } func (c validationCost) add(other validationCost) validationCost { return validationCost{c.extraRuns + other.extraRuns, c.punctuation + other.punctuation} } func (c validationCost) arrayTokens() int { return 1 + c.extraRuns + (c.punctuation+2)/2 // opening and closing brackets } func measureJSON(estimator *chunking.SimpleTokenEstimator, fragment []byte) validationCost { words, punctuation := estimator.CountComponents(string(fragment)) return validationCost{words - 1, punctuation} } func measureValidationItem(estimator *chunking.SimpleTokenEstimator, item LLMValidationItem) (validationCost, error) { // Every positive local reference has the same cost, regardless of digit count. item.CorrectionIndex = 1 payload, err := json.Marshal(item) if err != nil { return validationCost{}, fmt.Errorf("marshal batch payload: %w", err) } return measureJSON(estimator, payload), nil } type fittedValidationItem struct { item LLMValidationItem cost validationCost tokens int } // fitValidationContext tries the required payload followed by glossary entries // and then neighboring segments. The first excess ends the usable prefix. func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) { fitted := fittedValidationItem{item: item} fitted.item.GlossaryEntries = item.GlossaryEntries[:0] fitted.item.NeighboringSegments = item.NeighboringSegments[:0] simple, _ := estimator.(*chunking.SimpleTokenEstimator) var err error if simple != nil { fitted.cost, err = measureValidationItem(simple, fitted.item) fitted.tokens = fitted.cost.arrayTokens() } else { fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item}) } if err != nil || fitted.tokens > budget { return fitted, err } try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) { cost := fitted.cost var tokens int if simple != nil { payload, err := json.Marshal(entry) if err != nil { return false, fmt.Errorf("marshal validation %s: %w", field, err) } if first { cost = cost.add(measureJSON(simple, []byte(`,"`+field+`":[]`))) } else { cost.punctuation++ // separator between entries } cost = cost.add(measureJSON(simple, payload)) tokens = cost.arrayTokens() } else { var err error tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{candidate}) if err != nil { return false, err } } if tokens > budget { return false, nil } fitted = fittedValidationItem{item: candidate, cost: cost, tokens: tokens} return true, nil } for i, entry := range item.GlossaryEntries { candidate := fitted.item candidate.GlossaryEntries = item.GlossaryEntries[:i+1] if ok, err := try(candidate, "glossary_entries", i == 0, entry); !ok { return fitted, err } } for i, entry := range item.NeighboringSegments { candidate := fitted.item candidate.NeighboringSegments = item.NeighboringSegments[:i+1] if ok, err := try(candidate, "neighboring_segments", i == 0, entry); !ok { return fitted, err } } return fitted, nil }