package validators import ( "context" "encoding/json" "fmt" "slices" "sort" "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" ) type LLMBackedValidator struct { name string validatorType LLMValidatorType estimator chunking.TokenEstimator } func (v *LLMBackedValidator) Name() string { return v.name } // SetTokenEstimator allows deterministic test control over batching behavior. func (v *LLMBackedValidator) SetTokenEstimator(estimator chunking.TokenEstimator) { if v == nil || estimator == nil { return } v.estimator = estimator } func NewLLMBackedValidator(name string, validatorType LLMValidatorType) (*LLMBackedValidator, error) { if validatorPromptMetadata(validatorType).PromptID == "" { return nil, fmt.Errorf("unsupported LLM validator type %q", validatorType) } if strings.TrimSpace(name) == "" { return nil, fmt.Errorf("validator name must not be empty") } return &LLMBackedValidator{ name: name, validatorType: validatorType, estimator: chunking.NewSimpleTokenEstimator(), }, nil } func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, error) { if v == nil { return Result{}, fmt.Errorf("validator is nil") } if len(req.CandidateProposal) == 0 { return Result{ValidatorName: v.name, Decisions: nil}, nil } if req.LLMClient == nil { return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name) } items, immediate := buildValidationItems(req) if len(items) == 0 { all := append([]Decision(nil), immediate...) sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { return Result{}, err } return Result{ValidatorName: v.name, Decisions: all}, nil } maxTokens := config.DefaultValidationMaxPromptTokens if req.Settings.Validation.MaxPromptTokens > 0 { maxTokens = req.Settings.Validation.MaxPromptTokens } batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator) operations := make([]outcomes.Operation, 0) llmDecisions := make([]Decision, 0) partialResult := func() Result { decisions := append([]Decision(nil), immediate...) decisions = append(decisions, oversized...) decisions = append(decisions, llmDecisions...) return Result{ValidatorName: v.name, Decisions: decisions, Operations: operations} } sectionIndex := 0 if req.SectionIndex != nil { sectionIndex = *req.SectionIndex } operationRef := func(kind outcomes.OperationKind, batchIndex *int) outcomes.Reference { return outcomes.Reference{ StageIndex: req.StageIndex, StageID: req.StageID, ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndex, Kind: kind, ValidatorName: v.name, ValidatorOrder: req.ValidatorOrder, BatchIndex: batchIndex, } } if len(oversized) > 0 { indices := make([]int, len(oversized)) for i, decision := range oversized { indices[i] = decision.ProposalIndex } operations = append(operations, outcomes.Operation{ Reference: operationRef(outcomes.OperationValidationInput, nil), FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: indices, }) } if err != nil { return partialResult(), err } for _, batch := range batches { if err := ctx.Err(); err != nil { return partialResult(), err } transcriptDescription := "" transcriptDescription = req.Settings.Context.Description variables, err := BuildValidationVariables(batch.Items, transcriptDescription) if err != nil { return partialResult(), err } var response LLMValidationResponse var completion StructuredCompletionResponse promptMetadata := validatorPromptMetadata(v.validatorType) coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt} completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{ PromptID: promptMetadata.PromptID, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, Context: coordinates, }, &response) batchIndex := batch.BatchIndex operation := outcomes.Operation{ Reference: operationRef(outcomes.OperationValidation, &batchIndex), Counts: completion.Execution, } writeInteraction := func(reconciled *validationReconciliation, callErr error) structuredoutput.InteractionArtifacts { if req.Interactions == nil { return structuredoutput.InteractionArtifacts{} } artifacts, _ := structuredoutput.WriteIndexedInteraction(req.Interactions, func(offset int, _ func(string) string) structuredoutput.Interaction { coordinates := coordinates coordinates.ProposalIndex = structuredoutput.OffsetProposalIndex(req.RepairProposalIndex, offset) stageInstance := req.ModuleInstance if req.SectionIndex != nil { stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex) } if coordinates.ProposalIndex != nil && req.RepairAttempt != nil { stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *coordinates.ProposalIndex, *req.RepairAttempt) } responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model} if reconciled != nil { view := *reconciled view.Decisions = slices.Clone(view.Decisions) for i := range view.Decisions { view.Decisions[i].ProposalIndex += offset view.Decisions[i].DiagnosticArtifactPath = "" } view.UnavailableIndices = slices.Clone(view.UnavailableIndices) for i := range view.UnavailableIndices { view.UnavailableIndices[i] += offset } responsePayload["reconciliation"] = view } indices := batchProposalIndices(batch.Items) for i := range indices { indices[i] += offset } return structuredoutput.Interaction{ Name: stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex), Context: coordinates, RequestMetadata: map[string]any{ "stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey, "validator_name": v.name, "validator_type": v.validatorType, "batch_index": batch.BatchIndex, "proposal_indices": indices, "repair_proposal_index": coordinates.ProposalIndex, "repair_attempt": req.RepairAttempt, "prompt": completion.Prompt, "response_schema": responseSchema.DiagnosticsMap(), }, RequestPayload: completion.Prepared, ResponsePayload: responsePayload, ErrorPayload: errPayload(callErr), } }) return artifacts } if err != nil { artifacts := writeInteraction(nil, err) structuredoutput.RecordFailure(&operation, err) operation.ProposalIndices = batchProposalIndices(batch.Items) operations = append(operations, operation) if operation.FailureClass == outcomes.FailureRecoverable { decisions := rejectBatch(batch.Items, string(operation.FailureReason), "model validation was unavailable") for i := range decisions { decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath } llmDecisions = append(llmDecisions, decisions...) continue } return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) } reconciled := reconcileLLMResponse(batch.Items, response) artifacts := writeInteraction(&reconciled, nil) if len(reconciled.UnavailableIndices) > 0 { operation.FailureClass = outcomes.FailureRecoverable operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid operation.ProposalIndices = reconciled.UnavailableIndices } else { operation.Succeeded = true operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution) if len(reconciled.IgnoredIndices) > 0 { operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered) } } operations = append(operations, operation) for i := range reconciled.Decisions { reconciled.Decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath } llmDecisions = append(llmDecisions, reconciled.Decisions...) } result := partialResult() all := result.Decisions if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { return result, err } sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) return result, nil } func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata { switch validatorType { case LLMValidatorTypeSpokenFormPlausibility: return prompts.MustLookupMetadata(prompts.PromptIDValidatorSpokenFormPlausibility) case LLMValidatorTypeMeaningReversal: return prompts.MustLookupMetadata(prompts.PromptIDValidatorMeaningReversalReview) case LLMValidatorTypeEditorialReview: return prompts.MustLookupMetadata(prompts.PromptIDValidatorEditorialReview) default: return prompts.Metadata{} } } func errPayload(err error) any { if err == nil { return nil } return map[string]any{"error": err.Error()} } func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) { items := make([]LLMValidationItem, 0, len(req.CandidateProposal)) immediate := make([]Decision, 0) req.Section = req.sectionLookup() segments := req.Section.segments var glossary glossaryContext if req.GlossaryView != nil { glossary = req.GlossaryView.context } else { glossary = prepareGlossaryContext(req.Glossary) } for _, p := range req.CandidateProposal { position, ok := req.Section.Position(p.TargetSegmentID) if !ok { immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found")) continue } seg := segments[position] preview := req.Preview(p) if !preview.Applicable { immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation")) continue } neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText) items = append(items, LLMValidationItem{ CorrectionIndex: p.ProposalIndex, Speaker: seg.Speaker, OriginalText: p.OriginalText, CorrectedText: p.CorrectedText, OriginalSegmentText: seg.Text, CorrectedSegmentText: preview.CorrectedSegmentText, Categories: append([]string(nil), seg.Categories...), NeighboringSegments: neighbors, GlossaryEntries: glossaryEntries, }) } return items, immediate } type validationReconciliation struct { Decisions []Decision `json:"decisions"` UnavailableIndices []int `json:"unavailable_indices,omitempty"` IgnoredIndices []json.Number `json:"ignored_indices,omitempty"` } func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation { result := validationReconciliation{Decisions: make([]Decision, 0, len(items))} seen := make(map[int][]LLMValidationDecision, len(items)) for _, d := range response.Validations { index, ok := jsonnumber.PositiveIndex(d.CorrectionIndex.String(), len(items)) if !ok { result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex) continue } seen[index] = append(seen[index], d) } for i, item := range items { entries := seen[i+1] if len(entries) == 0 { result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing")) result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex) continue } if len(entries) > 1 { result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonDuplicateValidationDecision, "required model validation decision appeared more than once")) result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex) continue } d := entries[0] reasonCode := ReasonApproved if !d.Approved { reasonCode = "llm_rejected" } result.Decisions = append(result.Decisions, Decision{ ProposalIndex: item.CorrectionIndex, Approved: d.Approved, Disposition: decisionDisposition(d.Approved), ReasonCode: reasonCode, Message: strings.TrimSpace(d.Reason), }) } return result } func oversizedValidationDecision(index int) Decision { return unavailable(index, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens") } func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision { out := make([]Decision, 0, len(items)) for _, item := range items { out = append(out, unavailable(item.CorrectionIndex, reasonCode, message)) } return out } func batchProposalIndices(items []LLMValidationItem) []int { indices := make([]int, len(items)) for i, item := range items { indices[i] = item.CorrectionIndex } return indices } func decisionDisposition(approved bool) DecisionDisposition { if approved { return DispositionApproved } return DispositionRejected }