2026-11-22 03:51:08 +00:00
package cli
import (
"bytes"
2026-11-22 04:05:44 +00:00
"encoding/json"
2026-11-22 03:51:08 +00:00
"errors"
2026-11-22 04:05:44 +00:00
"fmt"
"net/http"
"net/http/httptest"
2026-11-22 03:51:08 +00:00
"os"
"os/exec"
"path/filepath"
"runtime"
2026-11-22 04:05:44 +00:00
"strings"
2026-11-22 03:51:08 +00:00
"testing"
2026-11-22 04:05:44 +00:00
"go.maximumdirect.net/commandkit"
2026-11-22 03:51:08 +00:00
)
2026-11-22 04:05:44 +00:00
func buildNativeExecutable ( t * testing . T ) string {
t . Helper ( )
2026-11-22 03:51:08 +00:00
binary := filepath . Join ( t . TempDir ( ) , "notarius" )
build := exec . Command ( filepath . Join ( runtime . GOROOT ( ) , "bin" , "go" ) , "build" , "-o" , binary , "../../cmd/notarius" )
build . Env = append ( os . Environ ( ) , "GOWORK=off" )
if output , err := build . CombinedOutput ( ) ; err != nil {
t . Fatalf ( "build executable: %v\n%s" , err , output )
}
2026-11-22 04:05:44 +00:00
return binary
}
func TestExecutableRunRoutesDeclareTheSameRequiredInputs ( t * testing . T ) {
binary := buildNativeExecutable ( t )
2026-11-22 03:51:08 +00:00
for _ , route := range [ ] [ ] string { { "run" } , { "pipeline" , "run" } } {
help := exec . Command ( binary , append ( append ( [ ] string ( nil ) , route ... ) , "--help" ) ... )
output , err := help . CombinedOutput ( )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "pipeline-id" ) ) || ! bytes . Contains ( output , [ ] byte ( "--input-file" ) ) {
t . Fatalf ( "%v help: %v\n%s" , route , err , output )
}
missingInput := exec . Command ( binary , append ( append ( [ ] string ( nil ) , route ... ) , "demo" ) ... )
output , err = missingInput . CombinedOutput ( )
var exit * exec . ExitError
if ! errors . As ( err , & exit ) || exit . ExitCode ( ) != 2 || ! bytes . Contains ( output , [ ] byte ( "--input-file" ) ) {
t . Fatalf ( "%v missing input: %v\n%s" , route , err , output )
}
}
}
2026-11-22 04:05:44 +00:00
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks ( t * testing . T ) {
binary := buildNativeExecutable ( t )
for _ , fallback := range [ ] bool { false , true } {
server := httptest . NewServer ( http . HandlerFunc ( func ( writer http . ResponseWriter , request * http . Request ) {
if fallback {
writer . WriteHeader ( http . StatusServiceUnavailable )
return
}
writer . Header ( ) . Set ( "Content-Type" , "application/json" )
_ = json . NewEncoder ( writer ) . Encode ( map [ string ] any { "id" : "local-fake" , "object" : "chat.completion" , "choices" : [ ] any { map [ string ] any { "index" : 0 , "message" : map [ string ] any { "role" : "assistant" , "content" : ` { "spell_casts":[]} ` } , "finish_reason" : "stop" } } } )
} ) )
t . Cleanup ( server . Close )
for _ , format := range [ ] commandkit . Format { commandkit . FormatText , commandkit . FormatJSON , commandkit . FormatYAML } {
t . Run ( fmt . Sprintf ( "%s/fallback=%t" , format , fallback ) , func ( t * testing . T ) {
root := t . TempDir ( )
configuration := fmt . Sprintf ( ` version : 6
output : { directory : output }
pipelines :
demo :
input : { module : seriatim }
chunk : { module : generic }
llm_profile : local - fake
mode : permissive
max_additional_turns : 0
artifacts :
spells :
extract : { module : dnd / spells }
promptkit :
profiles :
- id : local - fake
endpoint : % q
model : fake - model
api_key_required : false
` , server . URL + "/v1" )
file := filepath . Join ( root , "config.yml" )
input := filepath . Join ( root , "input.json" )
for path , content := range map [ string ] string { file : configuration , input : ` { "metadata": { "id":"report-source"},"segments":[ { "id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]} ` } {
if err := os . WriteFile ( path , [ ] byte ( content ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
}
command := exec . Command ( binary , "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--llm-transient-retries=0" , "--format" , string ( format ) , "--verbose" )
var output , diagnostics bytes . Buffer
command . Stdout , command . Stderr = & output , & diagnostics
if err := command . Run ( ) ; err != nil {
t . Fatalf ( "subprocess: %v stdout=%s stderr=%s" , err , output . String ( ) , diagnostics . String ( ) )
}
if format == commandkit . FormatText {
if ! strings . Contains ( output . String ( ) , "produced 1 artifacts" ) || fallback != strings . Contains ( output . String ( ) , "Success with fallback:" ) {
t . Fatalf ( "completion text: %s" , output . String ( ) )
}
} else {
report := decodeCompletionReport ( t , format , output . Bytes ( ) )
if report [ "artifact_count" ] != float64 ( 1 ) || report [ "pipeline" ] != "demo" || fallback && report [ "fallback_count" ] != float64 ( 1 ) || ! fallback && report [ "fallback_count" ] != nil {
t . Fatalf ( "completion report: %#v" , report )
}
if _ , err := os . Stat ( filepath . Join ( report [ "output_directory" ] . ( string ) , "lanes" , "spells.json" ) ) ; err != nil {
t . Fatal ( err )
}
}
} )
}
}
}