2026-05-12 02:25:33 +00:00
package proposal_generation
import (
"context"
2026-05-13 02:10:24 +00:00
"encoding/json"
2026-05-12 02:25:33 +00:00
"errors"
"os"
"path/filepath"
"strings"
"testing"
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
2026-11-16 03:43:29 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
2026-05-12 02:25:33 +00:00
)
type fakeStructuredClient struct {
2026-09-24 17:09:35 +00:00
responses [ ] StructuredCorrectionSet
err error
completion contracts . StructuredCompletionResponse
calls [ ] contracts . StructuredCompletionRequest
2026-05-12 02:25:33 +00:00
}
func ( f * fakeStructuredClient ) CompleteStructured ( ctx context . Context , req contracts . StructuredCompletionRequest , out any ) ( contracts . StructuredCompletionResponse , error ) {
_ = ctx
f . calls = append ( f . calls , req )
if f . err != nil {
2026-09-24 17:09:35 +00:00
return f . completion , f . err
2026-05-12 02:25:33 +00:00
}
target , ok := out . ( * StructuredCorrectionSet )
if ! ok {
return contracts . StructuredCompletionResponse { } , errors . New ( "unexpected output type" )
}
if len ( f . responses ) == 0 {
return contracts . StructuredCompletionResponse { } , errors . New ( "unexpected call" )
}
* target = f . responses [ 0 ]
f . responses = f . responses [ 1 : ]
2026-09-24 02:03:08 +00:00
var content strings . Builder
for _ , value := range req . Variables {
content . WriteString ( value )
}
2026-09-24 17:09:35 +00:00
response := f . completion
response . Prepared = & structuredoutput . PreparedSnapshot { Messages : [ ] structuredoutput . Message { { Role : "user" , Content : content . String ( ) } } }
return response , nil
2026-05-12 02:25:33 +00:00
}
2026-05-13 02:10:24 +00:00
type captureDiagnosticsWriter struct {
2026-09-24 21:48:18 +00:00
err error
2026-05-13 02:10:24 +00:00
lastStage string
lastRequestMetadata any
lastRequestPayload any
}
2026-11-16 03:43:29 +00:00
func ( w * captureDiagnosticsWriter ) WriteInteraction ( interaction structuredoutput . Interaction ) ( structuredoutput . InteractionArtifacts , error ) {
w . lastStage = interaction . Name
w . lastRequestMetadata = interaction . RequestMetadata
w . lastRequestPayload = interaction . RequestPayload
return structuredoutput . InteractionArtifacts { } , w . err
2026-05-13 02:10:24 +00:00
}
2026-05-12 02:25:33 +00:00
func defaultRequest ( t * testing . T ) Request {
t . Helper ( )
return Request {
ModuleKey : "test_module" ,
ModuleInstance : "test_module" ,
ReplacementPolicy : proposals . ReplacementPolicyRequireUnique ,
2026-09-24 02:03:08 +00:00
PromptID : "modules.grammar.proposal" ,
Variables : map [ string ] string { "SectionJSON" : "[]" } ,
2026-09-25 12:38:23 +00:00
WorkingTranscript : & schema . Transcript { Segments : [ ] schema . Segment { { ID : 7 , Text : "teh" } , { ID : 25 , Text : "other" } } } ,
2026-05-12 02:25:33 +00:00
}
}
func TestGenerateCandidatesSuccess ( t * testing . T ) {
client := & fakeStructuredClient {
responses : [ ] StructuredCorrectionSet {
{
Corrections : [ ] StructuredCorrectionProposal {
2026-11-11 17:05:52 +00:00
{ SegmentReference : 1 , OriginalText : "teh" , CorrectedText : "the" } ,
2026-05-12 02:25:33 +00:00
} ,
} ,
} ,
}
section := contracts . SectionMetadata { Index : 2 }
req := defaultRequest ( t )
req . LLMClient = client
req . Section = & section
got , err := GenerateCandidates ( context . Background ( ) , req )
if err != nil {
t . Fatalf ( "GenerateCandidates error: %v" , err )
}
2026-11-16 14:30:57 +00:00
if len ( got . Proposals ) != 1 || len ( got . SegmentReferences ) != 1 || got . SegmentReferences [ 0 ] != 1 {
2026-05-12 02:25:33 +00:00
t . Fatalf ( "unexpected proposal lengths: %+v" , got )
}
2026-11-16 14:30:57 +00:00
if got . Proposals [ 0 ] . TargetSegmentID != 7 {
t . Fatalf ( "local reference not resolved: %+v" , got . Proposals )
2026-05-12 02:25:33 +00:00
}
2026-09-24 17:09:35 +00:00
if len ( got . Operations ) != 1 || ! got . Operations [ 0 ] . Succeeded || got . Operations [ 0 ] . Reference . SectionIndex != 2 || got . Operations [ 0 ] . Reference . Kind != outcomes . OperationProposal {
t . Fatalf ( "proposal operation identity: %+v" , got . Operations )
}
2026-05-12 02:25:33 +00:00
}
2026-05-13 02:10:24 +00:00
func TestGenerateCandidatesUsesCorrectionSetSchema ( t * testing . T ) {
client := & fakeStructuredClient {
responses : [ ] StructuredCorrectionSet {
2026-11-11 17:05:52 +00:00
{ Corrections : [ ] StructuredCorrectionProposal { { SegmentReference : 1 , OriginalText : "teh" , CorrectedText : "the" } } } ,
2026-05-13 02:10:24 +00:00
} ,
}
req := defaultRequest ( t )
req . LLMClient = client
_ , err := GenerateCandidates ( context . Background ( ) , req )
if err != nil {
t . Fatalf ( "GenerateCandidates error: %v" , err )
}
if len ( client . calls ) != 1 {
t . Fatalf ( "expected 1 LLM call, got %d" , len ( client . calls ) )
}
call := client . calls [ 0 ]
2026-09-24 02:03:08 +00:00
if call . Schema != responseschema . CorrectionSetKey || call . PromptID != req . PromptID || call . Context . Role != "proposal" {
t . Fatalf ( "unexpected call contract: %+v" , call )
2026-05-13 02:10:24 +00:00
}
}
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata ( t * testing . T ) {
client := & fakeStructuredClient {
responses : [ ] StructuredCorrectionSet {
2026-11-11 17:05:52 +00:00
{ Corrections : [ ] StructuredCorrectionProposal { { SegmentReference : 1 , OriginalText : "teh" , CorrectedText : "the" } } } ,
2026-05-13 02:10:24 +00:00
} ,
}
diag := & captureDiagnosticsWriter { }
req := defaultRequest ( t )
req . LLMClient = client
2026-11-16 03:43:29 +00:00
req . Interactions = diag
2026-05-13 02:10:24 +00:00
_ , err := GenerateCandidates ( context . Background ( ) , req )
if err != nil {
t . Fatalf ( "GenerateCandidates error: %v" , err )
}
metadata , ok := diag . lastRequestMetadata . ( map [ string ] any )
if ! ok {
t . Fatalf ( "expected request metadata map, got %T" , diag . lastRequestMetadata )
}
schemaMap , ok := metadata [ "response_schema" ] . ( map [ string ] any )
if ! ok {
t . Fatalf ( "expected response_schema metadata map, got %T" , metadata [ "response_schema" ] )
}
want := responseschema . MustLookup ( responseschema . CorrectionSetKey )
if schemaMap [ "id" ] != want . ID || schemaMap [ "version" ] != want . Version || schemaMap [ "name" ] != want . Name || schemaMap [ "sha256" ] != want . SHA256 {
t . Fatalf ( "unexpected diagnostics schema metadata: got=%v want=%+v" , schemaMap , want )
}
2026-09-24 02:03:08 +00:00
2026-05-13 02:10:24 +00:00
}
2026-05-23 10:07:06 -05:00
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation ( t * testing . T ) {
2026-05-12 02:25:33 +00:00
client := & fakeStructuredClient {
responses : [ ] StructuredCorrectionSet {
{
Corrections : [ ] StructuredCorrectionProposal {
2026-11-16 04:09:25 +00:00
{ SegmentReference : 9 , OriginalText : "x" , CorrectedText : "" } ,
2026-05-12 02:25:33 +00:00
} ,
} ,
} ,
}
req := defaultRequest ( t )
req . LLMClient = client
2026-11-16 04:09:25 +00:00
req . Interactions = llm . NewDiagnosticsWriter ( t . TempDir ( ) )
2026-05-12 02:25:33 +00:00
2026-05-23 10:07:06 -05:00
result , err := GenerateCandidates ( context . Background ( ) , req )
if err != nil {
t . Fatalf ( "expected invalid correction to survive generation, got %v" , err )
}
2026-11-16 14:30:57 +00:00
if len ( result . Proposals ) != 1 {
2026-05-23 10:07:06 -05:00
t . Fatalf ( "expected one correction, got %+v" , result )
}
2026-11-16 14:30:57 +00:00
if result . Proposals [ 0 ] . TargetSegmentID != 0 {
t . Fatalf ( "unexpected preserved correction: %+v" , result . Proposals [ 0 ] )
2026-05-23 10:07:06 -05:00
}
2026-11-16 04:09:25 +00:00
raw , err := os . ReadFile ( result . Artifacts . ResponsePayloadPath )
if err != nil {
t . Fatal ( err )
}
var evidence struct {
Decoded StructuredCorrectionSet ` json:"decoded" `
}
if err := json . Unmarshal ( raw , & evidence ) ; err != nil {
t . Fatal ( err )
}
if len ( result . SegmentReferences ) != 1 || result . SegmentReferences [ 0 ] != 9 ||
len ( evidence . Decoded . Corrections ) != 1 || evidence . Decoded . Corrections [ 0 ] . SegmentReference != 9 {
t . Fatalf ( "lost model reference: result=%v artifact=%s" , result . SegmentReferences , raw )
}
2026-09-24 17:09:35 +00:00
if len ( result . Operations ) != 1 || ! result . Operations [ 0 ] . Succeeded {
t . Fatalf ( "invalid individual correction should remain a usable response: %+v" , result . Operations )
2026-05-23 10:07:06 -05:00
}
}
2026-09-24 17:09:35 +00:00
func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation ( t * testing . T ) {
client := & fakeStructuredClient { err : & structuredoutput . Failure { Class : structuredoutput . FailureRecoverable , Reason : outcomes . ReasonInvalidOutputExhausted , Message : "structured output remained invalid" } }
2026-05-23 10:07:06 -05:00
req := defaultRequest ( t )
req . LLMClient = client
result , err := GenerateCandidates ( context . Background ( ) , req )
if err != nil {
2026-09-24 17:09:35 +00:00
t . Fatalf ( "expected recoverable model operation, got %v" , err )
2026-05-23 10:07:06 -05:00
}
2026-11-16 14:30:57 +00:00
if len ( result . Proposals ) != 0 {
2026-05-23 10:07:06 -05:00
t . Fatalf ( "expected no proposals on malformed response, got %+v" , result )
}
2026-09-24 17:09:35 +00:00
if len ( result . Operations ) != 1 || result . Operations [ 0 ] . Succeeded || result . Operations [ 0 ] . FailureReason != outcomes . ReasonInvalidOutputExhausted {
t . Fatalf ( "expected one failed operation, got %+v" , result . Operations )
2026-05-23 10:07:06 -05:00
}
2026-09-24 17:09:35 +00:00
}
2026-11-16 14:27:54 +00:00
func TestGenerateCandidatesUsesInternalReasonForUnspecifiedRecoverableFailure ( t * testing . T ) {
cause := errors . New ( "underlying call failure" )
req := defaultRequest ( t )
req . LLMClient = & fakeStructuredClient { err : & structuredoutput . Failure { Class : outcomes . FailureRecoverable , Cause : cause } }
result , err := GenerateCandidates ( context . Background ( ) , req )
if err != nil || len ( result . Operations ) != 1 {
t . Fatalf ( "recoverable proposal returned operations=%+v error=%v" , result . Operations , err )
}
operation := result . Operations [ 0 ]
if operation . FailureClass != outcomes . FailureRecoverable || operation . FailureReason != outcomes . ReasonInternalFailure || ! errors . Is ( operation . Cause , cause ) {
t . Fatalf ( "operation lost typed failure facts: %+v" , operation )
}
}
2026-09-24 17:09:35 +00:00
func TestGenerateCandidatesFatalSetupPreservesCallFacts ( t * testing . T ) {
client := & fakeStructuredClient { err : & structuredoutput . Failure { Class : structuredoutput . FailureFatal , Reason : outcomes . ReasonSetupFailure , Message : "structured model setup failed" , Cause : errors . New ( "broken prompt" ) } , completion : structuredoutput . Response { Execution : outcomes . ExecutionCounts { Available : true , ProviderAttempts : 0 } } }
req := defaultRequest ( t )
req . LLMClient = client
result , err := GenerateCandidates ( context . Background ( ) , req )
if err == nil || len ( result . Operations ) != 1 || result . Operations [ 0 ] . FailureReason != outcomes . ReasonSetupFailure || ! result . Operations [ 0 ] . Counts . Available {
t . Fatalf ( "fatal setup lost call facts: %+v, %v" , result , err )
2026-05-12 02:25:33 +00:00
}
}
2026-09-24 17:09:35 +00:00
func TestGenerateCandidatesValidEmptyResponseIsSuccessful ( t * testing . T ) {
client := & fakeStructuredClient { responses : [ ] StructuredCorrectionSet { { } } }
2026-05-23 18:02:53 +00:00
req := defaultRequest ( t )
req . LLMClient = client
result , err := GenerateCandidates ( context . Background ( ) , req )
2026-11-16 14:30:57 +00:00
if err != nil || len ( result . Proposals ) != 0 || len ( result . Operations ) != 1 || ! result . Operations [ 0 ] . Succeeded {
2026-09-24 17:09:35 +00:00
t . Fatalf ( "valid empty response: %+v, %v" , result , err )
2026-05-23 18:02:53 +00:00
}
}
2026-11-16 03:37:29 +00:00
func TestGenerateCandidatesDiagnosticsWritten ( t * testing . T ) {
2026-05-12 02:25:33 +00:00
client := & fakeStructuredClient {
responses : [ ] StructuredCorrectionSet {
2026-11-16 03:37:29 +00:00
{ Corrections : [ ] StructuredCorrectionProposal { { SegmentReference : 1 , OriginalText : "source text" , CorrectedText : "safe" } } } ,
2026-05-12 02:25:33 +00:00
} ,
}
req := defaultRequest ( t )
req . LLMClient = client
2026-11-16 03:43:29 +00:00
dir := t . TempDir ( )
req . Interactions = llm . NewDiagnosticsWriter ( dir )
2026-11-16 03:37:29 +00:00
req . Variables = map [ string ] string { "SectionJSON" : "section text" }
2026-11-16 03:43:29 +00:00
req . Section = & contracts . SectionMetadata { Index : 3 }
2026-05-12 02:25:33 +00:00
got , err := GenerateCandidates ( context . Background ( ) , req )
if err != nil {
t . Fatalf ( "GenerateCandidates error: %v" , err )
}
if got . Artifacts . ResponsePayloadPath == "" || got . Artifacts . RequestPayloadPath == "" {
t . Fatalf ( "expected diagnostics artifact paths, got %+v" , got . Artifacts )
}
for _ , path := range [ ] string { got . Artifacts . RequestPayloadPath , got . Artifacts . ResponsePayloadPath } {
raw , readErr := os . ReadFile ( path )
if readErr != nil {
t . Fatalf ( "read artifact %q: %v" , path , readErr )
}
2026-11-16 03:37:29 +00:00
if ! json . Valid ( raw ) {
t . Fatalf ( "invalid JSON artifact %q: %s" , path , raw )
2026-05-12 02:25:33 +00:00
}
}
2026-05-13 02:10:24 +00:00
raw , readErr := os . ReadFile ( got . Artifacts . RequestMetadataPath )
if readErr != nil {
t . Fatalf ( "read metadata artifact %q: %v" , got . Artifacts . RequestMetadataPath , readErr )
}
var metadata map [ string ] any
if err := json . Unmarshal ( raw , & metadata ) ; err != nil {
t . Fatalf ( "unmarshal metadata artifact: %v" , err )
}
2026-11-16 14:30:57 +00:00
if start , ok := metadata [ "start_index" ] . ( float64 ) ; ! ok || start != 0 {
t . Fatalf ( "missing response-local origin: %s" , raw )
}
2026-05-13 02:10:24 +00:00
schemaMap , ok := metadata [ "response_schema" ] . ( map [ string ] any )
if ! ok {
t . Fatalf ( "expected response_schema metadata in diagnostics, got %T" , metadata [ "response_schema" ] )
}
want := responseschema . MustLookup ( responseschema . CorrectionSetKey )
if schemaMap [ "id" ] != want . ID || schemaMap [ "version" ] != want . Version || schemaMap [ "name" ] != want . Name || schemaMap [ "sha256" ] != want . SHA256 {
t . Fatalf ( "unexpected schema metadata in diagnostics: got=%v want=%+v" , schemaMap , want )
}
2026-11-16 03:43:29 +00:00
var coordinates struct {
Context structuredoutput . DiagnosticContext ` json:"context" `
}
if err := json . Unmarshal ( raw , & coordinates ) ; err != nil {
t . Fatal ( err )
}
if c := coordinates . Context ; c . Role != "proposal" || c . ModuleInstance != req . ModuleInstance || c . SectionIndex == nil || * c . SectionIndex != 3 {
t . Fatalf ( "proposal metadata: %s" , raw )
}
2026-05-12 02:25:33 +00:00
}
func TestGenerateCandidatesClientError ( t * testing . T ) {
client := & fakeStructuredClient { err : errors . New ( "boom" ) }
req := defaultRequest ( t )
req . LLMClient = client
2026-11-16 03:43:29 +00:00
dir := t . TempDir ( )
req . Interactions = llm . NewDiagnosticsWriter ( dir )
2026-05-12 02:25:33 +00:00
got , err := GenerateCandidates ( context . Background ( ) , req )
if err == nil || ! strings . Contains ( err . Error ( ) , "completion failed" ) {
t . Fatalf ( "expected completion failure, got %v" , err )
}
2026-11-16 14:30:57 +00:00
if len ( got . Operations ) != 1 || got . Operations [ 0 ] . FailureReason != outcomes . ReasonInternalFailure || len ( got . Proposals ) != 0 {
2026-09-24 17:14:33 +00:00
t . Fatalf ( "expected fatal call facts without proposals, got %+v" , got )
2026-05-12 02:25:33 +00:00
}
2026-11-16 03:43:29 +00:00
matches , globErr := filepath . Glob ( filepath . Join ( dir , req . ModuleInstance , "*error-payload.json" ) )
2026-05-12 02:25:33 +00:00
if globErr != nil {
t . Fatalf ( "glob error: %v" , globErr )
}
if len ( matches ) == 0 {
2026-11-16 03:43:29 +00:00
t . Fatalf ( "expected error diagnostics artifact under %s" , dir )
2026-05-12 02:25:33 +00:00
}
}
2026-09-24 21:48:18 +00:00
func TestProposalDiagnosticFailureDoesNotChangeOperation ( t * testing . T ) {
cause := errors . New ( "diagnostic storage unavailable" )
req := defaultRequest ( t )
req . LLMClient = & fakeStructuredClient { responses : [ ] StructuredCorrectionSet { { } } }
2026-11-16 03:43:29 +00:00
sink := & captureDiagnosticsWriter { err : cause }
req . Interactions = sink
2026-09-24 21:48:18 +00:00
result , err := GenerateCandidates ( context . Background ( ) , req )
2026-11-16 03:43:29 +00:00
if err != nil || sink . lastStage == "" || len ( result . Operations ) != 1 || ! result . Operations [ 0 ] . Succeeded || len ( result . Operations [ 0 ] . RecoveryReasons ) != 0 || result . Artifacts . ResponsePayloadPath != "" {
t . Fatalf ( "debug failure changed outcome: %+v %v" , result , err )
2026-09-24 21:48:18 +00:00
}
}