2026-08-11 21:21:33 +00:00
package usecase
import (
"context"
"errors"
"testing"
2026-11-02 14:38:00 +00:00
"go.maximumdirect.net/promptkit/internal/domain"
2026-11-21 14:12:34 +00:00
"go.maximumdirect.net/promptkit/internal/validate"
2026-08-11 21:21:33 +00:00
)
type outputContractTestCollaborators struct {
artifacts * fakeArtifactReader
renderer * fakeRenderer
llm * fakeLLM
validator * recordingValidationPreparer
admitter * fakeRunAdmitter
}
func newOutputContractTestRunner ( ) ( * Runner , outputContractTestCollaborators ) {
collaborators := outputContractTestCollaborators {
artifacts : defaultArtifactReader ( ) ,
renderer : defaultRenderer ( ) ,
llm : & fakeLLM { forbid : true } ,
validator : & recordingValidationPreparer { plan : & recordingPreparedValidation { } } ,
admitter : & fakeRunAdmitter { } ,
}
return NewRunner (
& fakePromptRepo { def : promptDef ( domain . FormatText , domain . ValidationNone , 0 ) } ,
& fakeExecutionProfileRepo { profiles : map [ string ] * domain . ExecutionProfile { "exec" : defaultExecutionProfile ( ) } } ,
2026-09-30 04:18:27 +00:00
2026-08-11 21:21:33 +00:00
collaborators . artifacts ,
collaborators . renderer ,
collaborators . llm ,
collaborators . validator ,
collaborators . admitter ,
) , collaborators
}
2026-11-20 15:20:29 +00:00
func TestResolveInlineSchemaOwnsRuntimeValue ( t * testing . T ) {
value := map [ string ] any { "properties" : map [ string ] any { "name" : map [ string ] any { "type" : "string" } } }
2026-11-21 14:12:34 +00:00
override := domain . OutputContractInput { Format : domain . FormatJSON , ValidationMode : domain . ValidationJSONSchema , Schema : value }
2026-11-20 15:20:29 +00:00
contract , err := resolveOutputContract ( context . Background ( ) , & domain . PromptDefinition { } , & override )
if err != nil {
t . Fatal ( err )
}
value [ "properties" ] . ( map [ string ] any ) [ "name" ] = false
if _ , ok := contract . Schema . ( map [ string ] any ) [ "properties" ] . ( map [ string ] any ) [ "name" ] . ( map [ string ] any ) ; ! ok {
t . Fatal ( "schema shares caller storage" )
}
var null map [ string ] any
override . Schema = null
override . SchemaPath = "schema.json"
contract , err = resolveOutputContract ( context . Background ( ) , & domain . PromptDefinition { } , & override )
if err != nil || contract . Schema != nil {
t . Fatalf ( "typed null selection: %#v, %v" , contract . Schema , err )
}
}
2026-08-11 21:21:33 +00:00
func TestRunnerPreparationNormalizesOutputContractConsistently ( t * testing . T ) {
tests := [ ] struct {
name string
2026-11-21 14:12:34 +00:00
override domain . OutputContractInput
2026-08-11 21:21:33 +00:00
want domain . OutputContract
} {
{
name : "empty replacement format defaults to text" ,
2026-11-21 14:12:34 +00:00
override : domain . OutputContractInput { ValidationMode : domain . ValidationNone } ,
2026-08-11 21:21:33 +00:00
want : domain . OutputContract { Format : domain . FormatText , ValidationMode : domain . ValidationNone } ,
} ,
{
name : "markdown basic replacement" ,
2026-11-21 14:12:34 +00:00
override : domain . OutputContractInput { Format : domain . FormatMarkdown , ValidationMode : domain . ValidationBasic } ,
want : domain . OutputContract { Format : domain . FormatMarkdown , ValidationMode : domain . ValidationBasic , RepairAttempts : 1 } ,
2026-08-11 21:21:33 +00:00
} ,
{
name : "json replacement preserves non-schema fields" ,
2026-11-21 14:12:34 +00:00
override : domain . OutputContractInput {
2026-08-11 21:21:33 +00:00
Format : domain . FormatJSON ,
ValidationMode : domain . ValidationJSON ,
SchemaPath : "ignored.json" ,
2026-11-21 14:12:34 +00:00
RepairAttempts : repairBudget ( 2 ) ,
2026-08-11 21:21:33 +00:00
} ,
want : domain . OutputContract {
Format : domain . FormatJSON ,
ValidationMode : domain . ValidationJSON ,
SchemaPath : "ignored.json" ,
RepairAttempts : 2 ,
} ,
} ,
}
for _ , tt := range tests {
t . Run ( tt . name , func ( t * testing . T ) {
runner , _ := newOutputContractTestRunner ( )
req := domain . RunRequest {
PromptID : "p" ,
ProfileID : "exec" ,
Inputs : singleInputRef ( ) ,
Validation : & tt . override ,
}
prepared , err := runner . Prepare ( context . Background ( ) , req )
if err != nil {
t . Fatalf ( "prepare: %v" , err )
}
preparedExecution , err := runner . PrepareExecution ( context . Background ( ) , req )
if err != nil {
t . Fatalf ( "prepare execution: %v" , err )
}
details := preparedExecution . Details ( )
if details == nil {
t . Fatal ( "prepared execution returned nil details" )
}
if prepared . OutputContract != tt . want || details . OutputContract != tt . want {
t . Fatalf ( "output contracts = (%+v, %+v), want %+v" , prepared . OutputContract , details . OutputContract , tt . want )
}
} )
}
}
func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion ( t * testing . T ) {
tests := [ ] struct {
name string
2026-11-21 14:12:34 +00:00
override domain . OutputContractInput
2026-08-11 21:21:33 +00:00
} {
2026-11-21 14:12:34 +00:00
{ name : "unsupported format" , override : domain . OutputContractInput { Format : "binary" , ValidationMode : domain . ValidationNone } } ,
{ name : "empty validation mode" , override : domain . OutputContractInput { Format : domain . FormatText } } ,
{ name : "negative repair attempts" , override : domain . OutputContractInput { Format : domain . FormatText , ValidationMode : domain . ValidationNone , RepairAttempts : repairBudget ( - 1 ) } } ,
{ name : "repair attempts above maximum" , override : domain . OutputContractInput { Format : domain . FormatText , ValidationMode : domain . ValidationBasic , RepairAttempts : repairBudget ( 4 ) } } ,
{ name : "none validation with repair attempts" , override : domain . OutputContractInput { Format : domain . FormatText , ValidationMode : domain . ValidationNone , RepairAttempts : repairBudget ( 1 ) } } ,
{ name : "json schema without path" , override : domain . OutputContractInput { Format : domain . FormatJSON , ValidationMode : domain . ValidationJSONSchema } } ,
2026-08-11 21:21:33 +00:00
}
for _ , tt := range tests {
for _ , operation := range [ ] string { "Prepare" , "PrepareExecution" } {
t . Run ( tt . name + "/" + operation , func ( t * testing . T ) {
runner , collaborators := newOutputContractTestRunner ( )
req := domain . RunRequest {
PromptID : "p" ,
ProfileID : "exec" ,
Inputs : singleInputRef ( ) ,
Validation : & tt . override ,
}
var err error
switch operation {
case "Prepare" :
var prepared * domain . PreparedRun
prepared , err = runner . Prepare ( context . Background ( ) , req )
if prepared != nil {
t . Fatalf ( "expected no partial prepared run, got %+v" , prepared )
}
case "PrepareExecution" :
var prepared * PreparedExecution
prepared , err = runner . PrepareExecution ( context . Background ( ) , req )
if prepared != nil {
t . Fatalf ( "expected no partial prepared execution, got %+v" , prepared )
}
default :
t . Fatalf ( "unknown operation %q" , operation )
}
if ! errors . Is ( err , ErrInvalidRequest ) {
t . Fatalf ( "expected ErrInvalidRequest, got %v" , err )
}
assertOutputContractCompletionSkipped ( t , collaborators )
} )
}
}
}
func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission ( t * testing . T ) {
runner , collaborators := newOutputContractTestRunner ( )
result , err := runner . Run ( context . Background ( ) , domain . RunRequest {
PromptID : "p" ,
ProfileID : "exec" ,
Inputs : singleInputRef ( ) ,
2026-11-21 14:12:34 +00:00
Validation : & domain . OutputContractInput {
2026-08-11 21:21:33 +00:00
Format : domain . OutputFormat ( "binary" ) ,
ValidationMode : domain . ValidationNone ,
} ,
} )
if result != nil {
t . Fatalf ( "expected no partial result, got %+v" , result )
}
if ! errors . Is ( err , ErrInvalidRequest ) {
t . Fatalf ( "expected ErrInvalidRequest, got %v" , err )
}
assertOutputContractCompletionSkipped ( t , collaborators )
}
2026-11-21 14:12:34 +00:00
func repairBudget ( value int ) * int { return & value }
func TestReplacementRepairExecution ( t * testing . T ) {
for _ , tc := range [ ] struct {
name string
budget * int
wantCalls int
wantStatus domain . ValidationStatus
} {
{ "default" , nil , 2 , domain . ValidationPassed } ,
{ "explicit zero" , repairBudget ( 0 ) , 1 , domain . ValidationFailed } ,
} {
t . Run ( tc . name , func ( t * testing . T ) {
client := & sequenceLLM { responses : [ ] * domain . GenerateResponse { { Content : "not json" } , { Content : ` { "ok":true} ` } } }
runner := NewRunnerWithRepairer ( & fakePromptRepo { def : promptDef ( domain . FormatText , domain . ValidationNone , 0 ) } ,
& fakeExecutionProfileRepo { profiles : map [ string ] * domain . ExecutionProfile { "exec" : defaultExecutionProfile ( ) } } ,
defaultArtifactReader ( ) , defaultRenderer ( ) , client , validate . NewValidator ( ) , NewDefaultOutputRepairer ( client ) , nil )
input := & domain . OutputContractInput { Format : domain . FormatJSON , ValidationMode : domain . ValidationJSON , RepairAttempts : tc . budget }
prepared , err := runner . PrepareExecution ( context . Background ( ) , domain . RunRequest { PromptID : "p" , ProfileID : "exec" , Inputs : singleInputRef ( ) , Validation : input } )
if err != nil {
t . Fatal ( err )
}
input . ValidationMode = domain . ValidationNone
input . RepairAttempts = repairBudget ( 3 )
result , err := runner . RunPrepared ( context . Background ( ) , prepared )
if err != nil || result . Validation . Status != tc . wantStatus || len ( client . requests ) != tc . wantCalls || result . Validation . RepairAttempts != tc . wantCalls - 1 {
t . Fatalf ( "result = %+v, calls = %d, error = %v" , result , len ( client . requests ) , err )
}
} )
}
}
func TestResolveReplacementRepairBudget ( t * testing . T ) {
configured := & domain . PromptDefinition { Validation : domain . OutputContract { Format : domain . FormatText , ValidationMode : domain . ValidationBasic , RepairAttempts : 3 } }
for _ , tc := range [ ] struct {
name string
input domain . OutputContractInput
want int
} {
{ "none" , domain . OutputContractInput { ValidationMode : domain . ValidationNone } , 0 } ,
{ "basic" , domain . OutputContractInput { ValidationMode : domain . ValidationBasic } , 1 } ,
{ "json" , domain . OutputContractInput { ValidationMode : domain . ValidationJSON } , 1 } ,
{ "schema" , domain . OutputContractInput { ValidationMode : domain . ValidationJSONSchema , Schema : true } , 1 } ,
{ "explicit zero" , domain . OutputContractInput { ValidationMode : domain . ValidationBasic , RepairAttempts : repairBudget ( 0 ) } , 0 } ,
} {
t . Run ( tc . name , func ( t * testing . T ) {
got , err := resolveOutputContract ( context . Background ( ) , configured , & tc . input )
if err != nil || got . RepairAttempts != tc . want {
t . Fatalf ( "contract = %+v, %v" , got , err )
}
} )
}
}
2026-08-11 21:21:33 +00:00
func assertOutputContractCompletionSkipped ( t * testing . T , collaborators outputContractTestCollaborators ) {
t . Helper ( )
if collaborators . artifacts . calls != 0 || collaborators . renderer . calls != 0 ||
2026-11-05 20:11:26 +00:00
collaborators . validator . prepareCalls != 0 ||
2026-08-11 21:21:33 +00:00
len ( collaborators . admitter . backendIDs ) != 0 || collaborators . llm . calls != 0 {
t . Fatalf (
2026-11-05 20:11:26 +00:00
"invalid output contract reached downstream work: artifacts=%d renderer=%d prepare_validation=%d admissions=%d generation=%d" ,
2026-08-11 21:21:33 +00:00
collaborators . artifacts . calls ,
collaborators . renderer . calls ,
collaborators . validator . prepareCalls ,
len ( collaborators . admitter . backendIDs ) ,
collaborators . llm . calls ,
)
}
}