Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
}
|
||||
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
|
||||
nextClient := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
|
||||
nextValidator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "ok"}}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
|
||||
}}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "99", Approved: true, Reason: "extra"},
|
||||
}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
|
||||
}}}
|
||||
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
|
||||
cause := errors.New("broken embedded schema")
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
|
||||
Reference in New Issue
Block a user