Validate internal decisions and reconcile exact model references

This commit is contained in:
2026-11-16 14:11:02 +00:00
parent 475557e4de
commit 0c144fd5a5
46 changed files with 544 additions and 376 deletions

View File

@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "ok"}}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "99", Approved: true, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
}}}
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})