2026-09-24 17:14:33 +00:00
package validators
import (
"context"
2026-09-25 12:38:23 +00:00
"encoding/json"
2026-09-24 17:14:33 +00:00
"errors"
"strings"
"testing"
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
2026-09-24 17:14:33 +00:00
)
type perItemEstimator struct { }
func ( perItemEstimator ) EstimateTokens ( payload string ) int {
return 100 * strings . Count ( payload , ` "correction_index" ` )
}
type validationReply struct {
response LLMValidationResponse
completion StructuredCompletionResponse
err error
}
type scriptedValidationClient struct {
replies [ ] validationReply
calls [ ] StructuredCompletionRequest
}
func ( c * scriptedValidationClient ) CompleteStructured ( _ context . Context , req StructuredCompletionRequest , target any ) ( StructuredCompletionResponse , error ) {
c . calls = append ( c . calls , req )
if len ( c . replies ) == 0 {
return StructuredCompletionResponse { } , errors . New ( "unexpected validation call" )
}
reply := c . replies [ 0 ]
c . replies = c . replies [ 1 : ]
if reply . err == nil {
* target . ( * LLMValidationResponse ) = reply . response
}
return reply . completion , reply . err
}
func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch ( t * testing . T ) {
candidates := [ ] proposals . EnrichedCorrectionProposal {
mk ( 0 , "gestures" , "Jesters" ) , mk ( 1 , "gestures" , "actors" ) , mk ( 2 , "gestures" , "figures" ) ,
}
2026-11-16 14:27:54 +00:00
cause := errors . New ( "underlying transport error" )
failed := & structuredoutput . Failure { Class : outcomes . FailureRecoverable , Reason : outcomes . ReasonTransportFailed , Message : "model transport failed" , Cause : cause }
2026-09-24 17:14:33 +00:00
client := & scriptedValidationClient { replies : [ ] validationReply {
2026-11-16 14:11:02 +00:00
{ response : LLMValidationResponse { Validations : [ ] LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } } ,
2026-09-24 17:14:33 +00:00
{ completion : StructuredCompletionResponse { Execution : outcomes . ExecutionCounts { Available : true , ProviderAttempts : 1 } } , err : failed } ,
2026-11-16 14:11:02 +00:00
{ response : LLMValidationResponse { Validations : [ ] LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } } ,
2026-09-24 17:14:33 +00:00
} }
validator , _ := NewLLMBackedValidator ( "editorial_review" , LLMValidatorTypeEditorialReview )
validator . SetTokenEstimator ( perItemEstimator { } )
req := makeReq ( candidates )
2026-11-10 18:13:12 +00:00
req . Settings . Validation . MaxPromptTokens = 150
2026-09-24 17:14:33 +00:00
req . LLMClient = client
result , err := validator . Validate ( context . Background ( ) , req )
if err != nil {
t . Fatal ( err )
}
if len ( result . Decisions ) != 3 || ! result . Decisions [ 0 ] . Approved || result . Decisions [ 1 ] . Disposition != DispositionUnavailable || ! result . Decisions [ 2 ] . Approved {
t . Fatalf ( "batch decisions: %+v" , result . Decisions )
}
if len ( result . Operations ) != 3 || result . Operations [ 1 ] . FailureReason != outcomes . ReasonTransportFailed || result . Operations [ 1 ] . Reference . BatchIndex == nil || * result . Operations [ 1 ] . Reference . BatchIndex != 1 || result . Operations [ 1 ] . Counts . ProviderAttempts != 1 {
t . Fatalf ( "batch operations: %+v" , result . Operations )
}
2026-11-16 14:27:54 +00:00
operation := result . Operations [ 1 ]
if operation . FailureClass != outcomes . FailureRecoverable || ! errors . Is ( operation . Cause , cause ) {
t . Fatalf ( "validation lost typed failure facts: %+v" , operation )
}
2026-09-24 17:14:33 +00:00
if err := EnforceDecisionCardinality ( candidates , result . Decisions ) ; err != nil {
t . Fatal ( err )
}
survivors := [ ] proposals . EnrichedCorrectionProposal { candidates [ 0 ] , candidates [ 2 ] }
nextClient := & scriptedValidationClient { replies : [ ] validationReply {
2026-11-16 14:11:02 +00:00
{ response : LLMValidationResponse { Validations : [ ] LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } } ,
{ response : LLMValidationResponse { Validations : [ ] LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } } ,
2026-09-24 17:14:33 +00:00
} }
nextValidator , _ := NewLLMBackedValidator ( "meaning_reversal_review" , LLMValidatorTypeMeaningReversal )
nextValidator . SetTokenEstimator ( perItemEstimator { } )
nextReq := makeReq ( survivors )
2026-11-10 18:13:12 +00:00
nextReq . Settings . Validation . MaxPromptTokens = 150
2026-09-24 17:14:33 +00:00
nextReq . LLMClient = nextClient
next , err := nextValidator . Validate ( context . Background ( ) , nextReq )
if err != nil || len ( next . Decisions ) != 2 || len ( nextClient . calls ) != 2 || * nextClient . calls [ 0 ] . Context . BatchIndex != 0 || * nextClient . calls [ 1 ] . Context . BatchIndex != 1 {
t . Fatalf ( "later validator rebatching: %+v, calls=%+v, err=%v" , next , nextClient . calls , err )
}
2026-09-25 12:38:23 +00:00
for _ , call := range append ( client . calls , nextClient . calls ... ) {
var items [ ] LLMValidationItem
if err := json . Unmarshal ( [ ] byte ( call . Variables [ "PayloadJSON" ] ) , & items ) ; err != nil {
t . Fatal ( err )
}
if len ( items ) != 1 || items [ 0 ] . CorrectionIndex != 1 {
t . Fatalf ( "batch references must restart: %+v" , items )
}
}
if ! next . Decisions [ 0 ] . Approved || ! next . Decisions [ 1 ] . Approved || next . Decisions [ 0 ] . ProposalIndex != 0 || next . Decisions [ 1 ] . ProposalIndex != 2 {
t . Fatalf ( "rebatching lost identities: %+v" , next . Decisions )
}
2026-09-24 17:14:33 +00:00
}
func TestValidationDomainInvalidAfterRepairIsAnError ( t * testing . T ) {
client := & scriptedValidationClient { replies : [ ] validationReply { {
2026-11-16 14:11:02 +00:00
response : LLMValidationResponse { Validations : [ ] LLMValidationDecision { { CorrectionIndex : "99" , Approved : true , Reason : "ok" } } } ,
2026-09-24 17:14:33 +00:00
completion : StructuredCompletionResponse { Execution : outcomes . ExecutionCounts { Available : true , ProviderAttempts : 2 , StructuralRepairs : 1 } } ,
} } }
validator , _ := NewLLMBackedValidator ( "editorial_review" , LLMValidatorTypeEditorialReview )
req := makeReq ( [ ] proposals . EnrichedCorrectionProposal { mk ( 0 , "gestures" , "Jesters" ) } )
req . LLMClient = client
result , err := validator . Validate ( context . Background ( ) , req )
2026-09-25 11:25:22 +00:00
if err != nil || len ( result . Decisions ) != 1 || result . Decisions [ 0 ] . ReasonCode != ReasonMissingValidationDecision || result . Decisions [ 0 ] . Disposition != DispositionUnavailable || len ( result . Operations ) != 1 || result . Operations [ 0 ] . FailureReason != outcomes . ReasonValidationDecisionsInvalid || result . Operations [ 0 ] . Counts . StructuralRepairs != 1 || len ( result . Operations [ 0 ] . RecoveryReasons ) != 0 || ! equalIndices ( result . Operations [ 0 ] . ProposalIndices , [ ] int { 0 } ) {
2026-09-24 17:14:33 +00:00
t . Fatalf ( "domain-invalid repaired response: %+v, %v" , result , err )
}
}
2026-09-25 11:25:22 +00:00
func TestValidationExtraDecisionsMergeActualRecoveries ( t * testing . T ) {
client := & scriptedValidationClient { replies : [ ] validationReply { {
response : LLMValidationResponse { Validations : [ ] LLMValidationDecision {
2026-11-16 14:11:02 +00:00
{ CorrectionIndex : "1" , Approved : true , Reason : "ok" } ,
{ CorrectionIndex : "99" , Approved : true , Reason : "extra" } ,
2026-09-25 11:25:22 +00:00
} } ,
completion : StructuredCompletionResponse { Execution : outcomes . ExecutionCounts { Available : true , ProviderAttempts : 3 , TransientRetries : 1 , StructuralRepairs : 1 } } ,
} } }
validator , _ := NewLLMBackedValidator ( "editorial_review" , LLMValidatorTypeEditorialReview )
req := makeReq ( [ ] proposals . EnrichedCorrectionProposal { mk ( 0 , "gestures" , "Jesters" ) } )
req . LLMClient = client
result , err := validator . Validate ( context . Background ( ) , req )
if err != nil || len ( result . Decisions ) != 1 || ! result . Decisions [ 0 ] . Approved || len ( result . Operations ) != 1 {
t . Fatalf ( "validation result: %+v, %v" , result , err )
}
op := result . Operations [ 0 ]
if ! op . Succeeded || op . Counts . ProviderAttempts != 3 || op . Counts . TransientRetries != 1 || op . Counts . StructuralRepairs != 1 || len ( op . RecoveryReasons ) != 3 || op . RecoveryReasons [ 0 ] != outcomes . RecoveryOutputRepaired || op . RecoveryReasons [ 1 ] != outcomes . RecoveryServiceRecovered || op . RecoveryReasons [ 2 ] != outcomes . RecoveryValidationDecisionsFiltered || len ( op . Filtered ) != 0 {
t . Fatalf ( "operation: %+v" , op )
}
}
2026-09-24 17:14:33 +00:00
func TestValidationFatalFailureKeepsEarlierFactsAndStops ( t * testing . T ) {
cause := errors . New ( "broken embedded schema" )
client := & scriptedValidationClient { replies : [ ] validationReply {
2026-11-16 14:11:02 +00:00
{ response : LLMValidationResponse { Validations : [ ] LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } } ,
2026-09-24 17:14:33 +00:00
{ err : & structuredoutput . Failure { Class : structuredoutput . FailureFatal , Reason : outcomes . ReasonSetupFailure , Message : "model setup failed" , Cause : cause } } ,
2026-11-16 14:11:02 +00:00
{ response : LLMValidationResponse { Validations : [ ] LLMValidationDecision { { CorrectionIndex : "1" , Approved : true , Reason : "ok" } } } } ,
2026-09-24 17:14:33 +00:00
} }
validator , _ := NewLLMBackedValidator ( "editorial_review" , LLMValidatorTypeEditorialReview )
validator . SetTokenEstimator ( perItemEstimator { } )
req := makeReq ( [ ] proposals . EnrichedCorrectionProposal { mk ( 0 , "gestures" , "Jesters" ) , mk ( 1 , "gestures" , "actors" ) , mk ( 2 , "gestures" , "figures" ) } )
2026-11-10 18:13:12 +00:00
req . Settings . Validation . MaxPromptTokens = 150
2026-09-24 17:14:33 +00:00
req . LLMClient = client
result , err := validator . Validate ( context . Background ( ) , req )
if ! errors . Is ( err , cause ) || len ( result . Decisions ) != 1 || len ( result . Operations ) != 2 || len ( client . calls ) != 2 {
t . Fatalf ( "fatal partial result: %+v, calls=%d, err=%v" , result , len ( client . calls ) , err )
}
}