Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -69,13 +69,14 @@ type WorkUnitOutcome struct {
|
||||
}
|
||||
|
||||
type ValidatorDecisionRecord struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Disposition validators.DecisionDisposition `json:"disposition"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
ExecutionClass validatormetadata.ExecutionClass `json:"-"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Disposition validators.DecisionDisposition `json:"disposition"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorRejectedChange struct {
|
||||
@@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct {
|
||||
Policy proposals.ReplacementPolicy
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
Validators []validators.Validator
|
||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
@@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
|
||||
var blocked, noEffect map[int]bool
|
||||
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
|
||||
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
|
||||
if feedback {
|
||||
blocked, noEffect = make(map[int]bool), make(map[int]bool)
|
||||
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
|
||||
@@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
|
||||
}
|
||||
class := validatormetadata.ClassOf(validator)
|
||||
submitted := eligible
|
||||
if feedback {
|
||||
submitted = feedbackCandidates(input, validator, noEffect, evidence)
|
||||
submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence)
|
||||
}
|
||||
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||
vResult, err := validator.Validate(ctx, validators.Request{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
WorkingTranscript: input.WorkingTranscript,
|
||||
Section: evidence.Section, Previews: evidence.Previews,
|
||||
@@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
byIndex[p.ProposalIndex] = p
|
||||
}
|
||||
|
||||
for _, d := range vResult.Decisions {
|
||||
if feedback && validator.Name() == "original_text_presence" && d.Approved {
|
||||
preview := evidence.Preview(byIndex[d.ProposalIndex])
|
||||
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
|
||||
d.Approved = false
|
||||
d.Disposition = validators.DispositionRejected
|
||||
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
|
||||
d.Message = "original_text matches multiple locations in the target segment"
|
||||
}
|
||||
dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions))
|
||||
for i, decision := range vResult.Decisions {
|
||||
disposition, contractErr := decision.EffectiveDisposition()
|
||||
if contractErr != nil {
|
||||
return validateSectionCandidatesResult{
|
||||
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
|
||||
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
|
||||
}
|
||||
if feedback && !d.Approved {
|
||||
dispositions[i] = disposition
|
||||
}
|
||||
for i, d := range vResult.Decisions {
|
||||
disposition := dispositions[i]
|
||||
approved := disposition == validators.DispositionApproved
|
||||
if feedback && !approved {
|
||||
blocked[d.ProposalIndex] = true
|
||||
if d.ReasonCode == validators.ReasonNoEffect {
|
||||
noEffect[d.ProposalIndex] = true
|
||||
@@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
}
|
||||
decisions = append(decisions, ValidatorDecisionRecord{
|
||||
ValidatorName: validator.Name(),
|
||||
ExecutionClass: class,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
Approved: d.Approved,
|
||||
Disposition: d.Disposition,
|
||||
Approved: approved,
|
||||
Disposition: disposition,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||
})
|
||||
if d.Approved {
|
||||
if approved {
|
||||
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
||||
continue
|
||||
}
|
||||
if d.Disposition != validators.DispositionUnavailable {
|
||||
if disposition != validators.DispositionUnavailable {
|
||||
if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok {
|
||||
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
|
||||
}
|
||||
@@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
TargetSegmentID: p.TargetSegmentID,
|
||||
OriginalText: p.OriginalText,
|
||||
CorrectedText: p.CorrectedText,
|
||||
Disposition: d.Disposition,
|
||||
Disposition: disposition,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
})
|
||||
@@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
|
||||
}
|
||||
}
|
||||
|
||||
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
|
||||
deterministic := make([]contracts.Validator, 0, len(in))
|
||||
llmBacked := make([]contracts.Validator, 0, len(in))
|
||||
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
|
||||
deterministic := make([]validators.Validator, 0, len(in))
|
||||
llmBacked := make([]validators.Validator, 0, len(in))
|
||||
for _, validator := range in {
|
||||
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
|
||||
class := validatormetadata.ClassOf(validator)
|
||||
validator = validatormetadata.Wrap(validator, class)
|
||||
if class == validatormetadata.ExecutionClassLLMBacked {
|
||||
llmBacked = append(llmBacked, validator)
|
||||
continue
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user