Validate internal decisions and reconcile exact model references

This commit is contained in:
2026-11-16 14:11:02 +00:00
parent 475557e4de
commit 0c144fd5a5
46 changed files with 544 additions and 376 deletions

View File

@@ -69,13 +69,14 @@ type WorkUnitOutcome struct {
}
type ValidatorDecisionRecord struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
ExecutionClass validatormetadata.ExecutionClass `json:"-"`
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidatorRejectedChange struct {
@@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct {
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
@@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
var blocked, noEffect map[int]bool
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
if feedback {
blocked, noEffect = make(map[int]bool), make(map[int]bool)
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
@@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
}
class := validatormetadata.ClassOf(validator)
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect, evidence)
submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence)
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
vResult, err := validator.Validate(ctx, validators.Request{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
Section: evidence.Section, Previews: evidence.Previews,
@@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
byIndex[p.ProposalIndex] = p
}
for _, d := range vResult.Decisions {
if feedback && validator.Name() == "original_text_presence" && d.Approved {
preview := evidence.Preview(byIndex[d.ProposalIndex])
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
d.Approved = false
d.Disposition = validators.DispositionRejected
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
d.Message = "original_text matches multiple locations in the target segment"
}
dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions))
for i, decision := range vResult.Decisions {
disposition, contractErr := decision.EffectiveDisposition()
if contractErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
}
if feedback && !d.Approved {
dispositions[i] = disposition
}
for i, d := range vResult.Decisions {
disposition := dispositions[i]
approved := disposition == validators.DispositionApproved
if feedback && !approved {
blocked[d.ProposalIndex] = true
if d.ReasonCode == validators.ReasonNoEffect {
noEffect[d.ProposalIndex] = true
@@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
Disposition: d.Disposition,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if d.Approved {
if approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue
}
if d.Disposition != validators.DispositionUnavailable {
if disposition != validators.DispositionUnavailable {
if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok {
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
}
@@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: d.Disposition,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
@@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
}
}
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
deterministic := make([]contracts.Validator, 0, len(in))
llmBacked := make([]contracts.Validator, 0, len(in))
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
deterministic := make([]validators.Validator, 0, len(in))
llmBacked := make([]validators.Validator, 0, len(in))
for _, validator := range in {
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
class := validatormetadata.ClassOf(validator)
validator = validatormetadata.Wrap(validator, class)
if class == validatormetadata.ExecutionClassLLMBacked {
llmBacked = append(llmBacked, validator)
continue
}