2026-09-13 16:51:46 +00:00
package cli
import (
"bytes"
2026-09-13 17:16:06 +00:00
"context"
2026-10-04 13:54:17 +00:00
"encoding/json"
2026-09-13 17:16:06 +00:00
"errors"
"io"
2026-09-13 16:51:46 +00:00
"path/filepath"
"strings"
"testing"
2026-09-13 17:16:06 +00:00
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
2026-09-13 16:51:46 +00:00
)
2026-11-18 00:29:56 +00:00
func TestExplainRejectsMalformedPointerBeforeLoadingEitherScope ( t * testing . T ) {
for _ , scope := range [ ] string { "notarius" , "promptkit" } {
t . Run ( scope , func ( t * testing . T ) {
var stdout , stderr strings . Builder
options := commandContractOptions ( t )
options . LookupEnv = func ( string ) ( string , bool ) {
t . Fatal ( "configuration discovery ran for invalid syntax" )
return "" , false
}
code := RunWithOptions ( [ ] string { "config" , "explain" , "--scope" , scope , "--config" , filepath . Join ( t . TempDir ( ) , "missing.yml" ) , "/profiles/~2" } , & stdout , & stderr , options )
if code != 2 || stdout . Len ( ) != 0 || ! strings . Contains ( stderr . String ( ) , "canonical JSON Pointer" ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout . String ( ) , stderr . String ( ) )
}
} )
}
}
2026-09-25 21:46:35 +00:00
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays ( t * testing . T ) {
2026-09-13 16:51:46 +00:00
t . Parallel ( )
root := t . TempDir ( )
path := filepath . Join ( root , "notarius.yml" )
writeConfigKitFile ( t , filepath . Join ( root , "fragment.yml" ) , "pipelines:\n imported:\n input: {module: input}\n" )
2026-09-25 21:46:35 +00:00
writeConfigKitFile ( t , filepath . Join ( root , "overlays" , "development.yml" ) , "pipelines:\n overlaid:\n input: {module: input}\n" )
2026-11-17 20:50:07 +00:00
writeConfigKitFile ( t , path , "version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: input}\n" )
2026-09-13 16:51:46 +00:00
var stdout , stderr bytes . Buffer
2026-09-25 21:46:35 +00:00
if code := RunWithOptions ( [ ] string { "pipelines" , "list" , "--config" , path , "--config-overlay" , "development" } , & stdout , & stderr , commandContractOptions ( t ) ) ; code != 0 || stdout . String ( ) != "base\nimported\noverlaid\n" {
2026-09-13 16:51:46 +00:00
t . Fatalf ( "list = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
2026-09-25 21:46:35 +00:00
if code := RunWithOptions ( [ ] string { "config" , "validate" , "--config" , path , "--config-overlay" , "development" } , & stdout , & stderr , commandContractOptions ( t ) ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "is valid" ) {
2026-09-13 16:51:46 +00:00
t . Fatalf ( "validate = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
2026-09-25 21:46:35 +00:00
for _ , selection := range [ ] [ ] string { nil , { "--config-overlay" , "" } , { "--config-overlay" , "default" } } {
stdout . Reset ( )
stderr . Reset ( )
args := append ( [ ] string { "pipelines" , "list" , "--config" , path } , selection ... )
if code := RunWithOptions ( args , & stdout , & stderr , commandContractOptions ( t ) ) ; code != 0 || stdout . String ( ) != "base\nimported\n" || stderr . Len ( ) != 0 {
t . Fatalf ( "base list %q = %d, %q, %q" , selection , code , stdout . String ( ) , stderr . String ( ) )
}
}
}
2026-10-03 18:08:07 +00:00
func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines ( t * testing . T ) {
t . Parallel ( )
path := writeCommandConfigContent ( t , ` version : 5
pipelines :
" parent " :
input :
module : seriatim
artifacts :
spells :
extract :
module : dnd / spells
structured_output_repair_attempts : 2
child :
extends : " parent "
structured_output_repair_attempts : 0
` )
options := commandContractOptions ( t )
var stdout , stderr bytes . Buffer
if code := RunWithOptions ( [ ] string { "pipelines" , "list" , "--config" , path } , & stdout , & stderr , options ) ; code != 0 || stdout . String ( ) != "child\nparent\n" || stderr . Len ( ) != 0 {
t . Fatalf ( "list = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "validate" , "--config" , path , "--pipeline" , "child" } , & stdout , & stderr , options ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , ` is valid for pipeline "child" ` ) || stderr . Len ( ) != 0 {
t . Fatalf ( "validate child = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "effective" , "--config" , path } , & stdout , & stderr , options ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "module: seriatim" ) || ! strings . Contains ( stdout . String ( ) , "structured_output_repair_attempts: 0" ) || strings . Contains ( stdout . String ( ) , "extends:" ) || stderr . Len ( ) != 0 {
t . Fatalf ( "effective = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "explain" , "--config" , path , "/pipelines/child/input/module" } , & stdout , & stderr , options ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , ` path: "/pipelines/child/input/module" ` ) || ! strings . Contains ( stdout . String ( ) , "parent" ) || stderr . Len ( ) != 0 {
t . Fatalf ( "explain inherited input = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
}
func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics ( t * testing . T ) {
t . Parallel ( )
for _ , test := range [ ] struct {
name string
content string
args [ ] string
want string
} {
{
name : "missing parent" ,
content : ` version : 5
pipelines :
child :
extends : absent
` ,
args : [ ] string { "config" , "validate" , "--pipeline" , "child" } ,
want : "inheritance.missing" ,
} ,
{
name : "cycle" ,
content : ` version : 5
pipelines :
first :
extends : second
second :
extends : first
` ,
args : [ ] string { "config" , "validate" , "--pipeline" , "first" } ,
want : "cycle" ,
} ,
{
name : "invalid directive type" ,
content : ` version : 5
pipelines :
child :
extends : 7
` ,
args : [ ] string { "config" , "validate" , "--pipeline" , "child" } ,
want : "extends" ,
} ,
{
name : "unselected missing parent" ,
content : ` version : 5
pipelines :
parent :
input :
module : seriatim
child :
extends : parent
unrelated :
extends : absent
` ,
args : [ ] string { "config" , "validate" , "--pipeline" , "child" } ,
want : "inheritance.missing" ,
} ,
{
name : "exact sibling mismatch" ,
content : ` version : 5
pipelines :
parent :
input :
module : seriatim
child :
extends : "parent "
` ,
args : [ ] string { "pipelines" , "list" } ,
want : "inheritance.missing" ,
} ,
} {
t . Run ( test . name , func ( t * testing . T ) {
t . Parallel ( )
path := writeCommandConfigContent ( t , test . content )
args := append ( [ ] string { } , test . args ... )
args = append ( args , "--config" , path )
var stdout , stderr bytes . Buffer
if code := RunWithOptions ( args , & stdout , & stderr , commandContractOptions ( t ) ) ; code != 1 || stdout . Len ( ) != 0 || stderr . Len ( ) == 0 || ! strings . Contains ( stderr . String ( ) , test . want ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout . String ( ) , stderr . String ( ) )
}
} )
}
}
2026-09-25 21:46:35 +00:00
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior ( t * testing . T ) {
t . Parallel ( )
for _ , args := range [ ] [ ] string {
{ "run" , "sample" , "--config-profile=development" } ,
{ "config" , "validate" , "--config-profile=development" } ,
{ "config" , "effective" , "--config-profile=development" } ,
{ "config" , "explain" , "--config-profile=development" , "" } ,
{ "pipelines" , "list" , "--config-profile=development" } ,
} {
t . Run ( strings . Join ( args [ : 2 ] , " " ) , func ( t * testing . T ) {
var stdout , stderr bytes . Buffer
if code := RunWithOptions ( args , & stdout , & stderr , commandContractOptions ( t ) ) ; code != 2 || stdout . Len ( ) != 0 || ! strings . Contains ( stderr . String ( ) , "not defined" ) {
t . Fatalf ( "args=%q code=%d stdout=%q stderr=%q" , args , code , stdout . String ( ) , stderr . String ( ) )
}
} )
}
2026-09-13 16:51:46 +00:00
}
func TestConfigKitReadOnlyCommandsRejectVersionFour ( t * testing . T ) {
t . Parallel ( )
path := filepath . Join ( t . TempDir ( ) , "notarius.yml" )
writeConfigKitFile ( t , path , "version: 4\n" )
var stdout , stderr bytes . Buffer
2026-09-13 17:12:56 +00:00
if code := runPipelinesList ( [ ] string { "--config" , path } , & stdout , & stderr , commandContractOptions ( t ) ) ; code != 1 || ! strings . Contains ( stderr . String ( ) , "version" ) {
2026-09-13 16:51:46 +00:00
t . Fatalf ( "list = %d, %q" , code , stderr . String ( ) )
}
}
2026-09-13 17:16:06 +00:00
func TestConfigurationInspectionCommandsRenderSafeConfigKitViews ( t * testing . T ) {
t . Parallel ( )
root := t . TempDir ( )
path := filepath . Join ( root , "notarius.yml" )
writeConfigKitFile ( t , path , ` version : 5
concurrency :
total_llm : 4
pipelines :
"0" :
input : { module : input }
` )
2026-09-25 21:46:35 +00:00
writeConfigKitFile ( t , filepath . Join ( root , "overlays" , "development.yml" ) , "concurrency:\n total_llm: 3\n" )
writeConfigKitFile ( t , filepath . Join ( root , "overlays" , "retired.yml" ) , "pipelines: null\n" )
2026-09-13 17:16:06 +00:00
providerCalled := false
opts := commandContractOptions ( t )
opts . LLMClientFactory = func ( context . Context , config . Config , string , LLMRuntimeOverrides ) ( contracts . StructuredLLMClient , [ ] artifacts . LLMProfileManifest , error ) {
providerCalled = true
return nil , nil , errors . New ( "LLM client must not be constructed" )
}
var stdout , stderr bytes . Buffer
2026-09-25 21:46:35 +00:00
if code := RunWithOptions ( [ ] string { "config" , "effective" , "--config" , path , "--config-overlay" , "development" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "total_llm: 3" ) || stderr . Len ( ) != 0 {
2026-09-13 17:16:06 +00:00
t . Fatalf ( "effective = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
if providerCalled {
t . Fatal ( "config effective constructed an LLM client" )
}
stdout . Reset ( )
stderr . Reset ( )
2026-09-25 21:46:35 +00:00
if code := RunWithOptions ( [ ] string { "config" , "explain" , "--config" , path , "--config-overlay" , "development" , "/concurrency/total_llm" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , ` path: "/concurrency/total_llm" ` ) || stderr . Len ( ) != 0 {
2026-09-13 17:16:06 +00:00
t . Fatalf ( "explain effective = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "explain" , "--config" , path , "" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , ` path: "" ` ) || stderr . Len ( ) != 0 {
t . Fatalf ( "explain root = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
2026-09-25 21:46:35 +00:00
if code := RunWithOptions ( [ ] string { "config" , "explain" , "--config" , path , "--config-overlay" , "retired" , "/pipelines/0" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , ` path: "/pipelines/0" ` ) || stderr . Len ( ) != 0 {
2026-09-13 17:16:06 +00:00
t . Fatalf ( "explain removed mapping key = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
}
func TestConfigExplainRejectsInvalidAndUnknownPointers ( t * testing . T ) {
t . Parallel ( )
path := writeCommandConfig ( t , "session" , "other" )
for _ , test := range [ ] struct {
name string
pointer string
wantCode int
} {
{ name : "missing" , wantCode : 2 } ,
{ name : "malformed escape" , pointer : "/pipelines~2session" , wantCode : 2 } ,
{ name : "unknown" , pointer : "/unknown" , wantCode : 1 } ,
{ name : "invalid sequence index" , pointer : "/pipelines/session/steps/+0" , wantCode : 1 } ,
} {
t . Run ( test . name , func ( t * testing . T ) {
var stdout , stderr bytes . Buffer
args := [ ] string { "config" , "explain" , "--config" , path }
if test . name != "missing" {
args = append ( args , test . pointer )
}
if code := RunWithOptions ( args , & stdout , & stderr , commandContractOptions ( t ) ) ; code != test . wantCode || stdout . Len ( ) != 0 || stderr . Len ( ) == 0 {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout . String ( ) , stderr . String ( ) )
}
} )
}
}
func TestConfigInspectionCommandsReportWriterFailures ( t * testing . T ) {
t . Parallel ( )
path := writeCommandConfig ( t , "session" , "other" )
var stderr bytes . Buffer
if code := RunWithOptions ( [ ] string { "config" , "effective" , "--config" , path } , failingInspectionWriter { } , & stderr , commandContractOptions ( t ) ) ; code != 1 || ! strings . Contains ( stderr . String ( ) , "write effective configuration" ) {
t . Fatalf ( "effective writer failure = %d, %q" , code , stderr . String ( ) )
}
}
2026-10-04 13:47:00 +00:00
func TestPromptKitScopedInspectionCommandsUseSelectedSources ( t * testing . T ) {
t . Parallel ( )
root := t . TempDir ( )
configPath := filepath . Join ( root , "notarius.yml" )
writeConfigKitFile ( t , configPath , "version: 5\npromptkit:\n profile_file: profiles/base.yml\n" )
writeConfigKitFile ( t , filepath . Join ( root , "overlays" , "development.yml" ) , "promptkit:\n profile_file: " + filepath . Join ( root , "profiles" , "imported.yml" ) + "\n" )
writeConfigKitFile ( t , filepath . Join ( root , "profiles" , "base.yml" ) , ` profiles :
base :
endpoint : https : //example.invalid/v1
model : base - model
extra_params :
Authorization : base - auth - secret
ordinary : visible - value
` )
writeConfigKitFile ( t , filepath . Join ( root , "profiles" , "imported.yml" ) , ` configkit :
imports :
- path : base . yml
profiles :
"escaped/id" :
model : imported - model
dnd - extraction :
model : imported - override
` )
assets , err := productionPromptAssets ( )
if err != nil {
t . Fatal ( err )
}
factoryCalls := 0
opts := commandContractOptions ( t )
opts . promptKitAssets = assets
opts . LLMClientFactory = func ( context . Context , config . Config , string , LLMRuntimeOverrides ) ( contracts . StructuredLLMClient , [ ] artifacts . LLMProfileManifest , error ) {
factoryCalls ++
return nil , nil , errors . New ( "inspection must not construct an LLM client" )
}
var stdout , stderr bytes . Buffer
if code := RunWithOptions ( [ ] string { "config" , "effective" , "--config" , configPath } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "pipelines:" ) || strings . Contains ( stdout . String ( ) , "profiles:" ) || stderr . Len ( ) != 0 {
t . Fatalf ( "default effective report = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
defaultReport := stdout . String ( )
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "effective" , "--scope" , "notarius" , "--config" , configPath } , & stdout , & stderr , opts ) ; code != 0 || stdout . String ( ) != defaultReport || stderr . Len ( ) != 0 {
t . Fatalf ( "explicit Notarius effective report = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "effective" , "--scope" , "promptkit" , "--config" , configPath , "--config-overlay" , "development" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "imported-model" ) || strings . Contains ( stdout . String ( ) , "base-auth-secret" ) || ! strings . Contains ( stdout . String ( ) , "visible-value" ) || stderr . Len ( ) != 0 {
t . Fatalf ( "PromptKit effective report = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "explain" , "--scope" , "promptkit" , "--config" , configPath , "--config-overlay" , "development" , "/profiles/escaped~1id/model" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "imported-model" ) || ! strings . Contains ( stdout . String ( ) , "imported.yml" ) || stderr . Len ( ) != 0 {
t . Fatalf ( "PromptKit imported explanation = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "explain" , "--scope" , "promptkit" , "--config" , configPath , "--config-overlay" , "development" , "/profiles/base/model" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "base-model" ) || ! strings . Contains ( stdout . String ( ) , "base.yml" ) || stderr . Len ( ) != 0 {
t . Fatalf ( "PromptKit imported base explanation = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
stdout . Reset ( )
stderr . Reset ( )
if code := RunWithOptions ( [ ] string { "config" , "explain" , "--scope" , "promptkit" , "--config" , configPath , "--config-overlay" , "development" , "/profiles/dnd-extraction/model" } , & stdout , & stderr , opts ) ; code != 0 || ! strings . Contains ( stdout . String ( ) , "imported-override" ) || ! strings . Contains ( stdout . String ( ) , "dnd-extraction.yaml" ) || stderr . Len ( ) != 0 {
t . Fatalf ( "PromptKit historical fallback explanation = %d, %q, %q" , code , stdout . String ( ) , stderr . String ( ) )
}
if factoryCalls != 0 {
t . Fatalf ( "inspection invoked LLM client factory %d times" , factoryCalls )
}
}
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors ( t * testing . T ) {
t . Parallel ( )
path := writeCommandConfig ( t , "session" , "other" )
opts := commandContractOptions ( t )
for _ , test := range [ ] struct {
name string
args [ ] string
want int
} {
{ name : "blank scope" , args : [ ] string { "config" , "effective" , "--scope=" } , want : 2 } ,
{ name : "unknown scope" , args : [ ] string { "config" , "effective" , "--scope" , "unknown" } , want : 2 } ,
{ name : "malformed pointer" , args : [ ] string { "config" , "explain" , "--scope" , "promptkit" , "--config" , path , "/profiles/~2" } , want : 2 } ,
{ name : "unknown pointer" , args : [ ] string { "config" , "explain" , "--scope" , "promptkit" , "--config" , path , "/not-configured" } , want : 1 } ,
} {
t . Run ( test . name , func ( t * testing . T ) {
var stdout , stderr bytes . Buffer
if code := RunWithOptions ( test . args , & stdout , & stderr , opts ) ; code != test . want || stderr . Len ( ) == 0 {
t . Fatalf ( "code=%d stdout=%q stderr=%q, want exit %d and diagnostic" , code , stdout . String ( ) , stderr . String ( ) , test . want )
}
} )
}
var stderr bytes . Buffer
if code := RunWithOptions ( [ ] string { "config" , "effective" , "--scope" , "promptkit" , "--config" , path } , failingInspectionWriter { } , & stderr , opts ) ; code != 1 || ! strings . Contains ( stderr . String ( ) , "write effective configuration" ) {
t . Fatalf ( "PromptKit writer failure = %d, %q" , code , stderr . String ( ) )
}
}
2026-10-04 13:54:17 +00:00
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup ( t * testing . T ) {
t . Parallel ( )
root := t . TempDir ( )
profilePath := filepath . Join ( root , "profiles.yml" )
writeConfigKitFile ( t , profilePath , ` backends :
managed :
endpoint : https : //provider.example.invalid/v1
api_key_env : NOT_SET_SYNTHETIC_PROFILE_KEY
profiles :
inherited :
backend : managed
model : inherited - model
temperature : 0.4
api_key_env : NOT_SET_SYNTHETIC_PROFILE_KEY
api_key_required : true
extra_params :
Authorization : private - authorization
ordinary : visible - value
selected :
extends : inherited
temperature : 0
extra_params : null
endpoint - only :
endpoint : https : //endpoint.example.invalid/v1
model : endpoint - model
api_key_required : false
" literal profile " :
endpoint : https : //literal.example.invalid/v1
model : literal - model
` )
configPath := filepath . Join ( root , "notarius.yml" )
writeConfigKitFile ( t , configPath , "version: 5\npromptkit:\n profile_file: " + profilePath + "\n" )
writeConfigKitFile ( t , filepath . Join ( root , "overlays" , "development.yml" ) , "promptkit:\n profile_file: " + profilePath + "\n" )
assets , err := productionPromptAssets ( )
if err != nil {
t . Fatal ( err )
}
lookupCalls := 0
factoryCalls := 0
opts := commandContractOptions ( t )
opts . promptKitAssets = assets
opts . LookupEnv = func ( string ) ( string , bool ) {
lookupCalls ++
return "" , false
}
opts . LLMClientFactory = func ( context . Context , config . Config , string , LLMRuntimeOverrides ) ( contracts . StructuredLLMClient , [ ] artifacts . LLMProfileManifest , error ) {
factoryCalls ++
return nil , nil , errors . New ( "profile inspection must not construct an LLM client" )
}
type profileReport struct {
ProfileID string ` json:"profile_id" `
APIKeyRequired bool ` json:"api_key_required" `
Target struct {
BackendID string ` json:"backend_id" `
Endpoint string ` json:"endpoint" `
Model string ` json:"model" `
Temperature float64 ` json:"temperature" `
APIKeyEnv string ` json:"api_key_env" `
ExtraParams map [ string ] any ` json:"extra_params" `
} ` json:"effective_model_params" `
}
run := func ( profileID string ) ( int , string , string , profileReport ) {
t . Helper ( )
var stdout , stderr bytes . Buffer
code := RunWithOptions ( [ ] string { "config" , "profile" , "--config" , configPath , "--config-overlay" , "development" , profileID } , & stdout , & stderr , opts )
var report profileReport
if code == 0 {
if bytes . Count ( stdout . Bytes ( ) , [ ] byte ( "\n" ) ) != 1 {
t . Fatalf ( "profile JSON output = %q, want one object and newline" , stdout . String ( ) )
}
if err := json . Unmarshal ( stdout . Bytes ( ) , & report ) ; err != nil {
t . Fatalf ( "decode profile JSON %q: %v" , stdout . String ( ) , err )
}
}
return code , stdout . String ( ) , stderr . String ( ) , report
}
code , output , diagnostic , selected := run ( "selected" )
if code != 0 || diagnostic != "" || selected . ProfileID != "selected" || ! selected . APIKeyRequired || selected . Target . BackendID != "managed" || selected . Target . Endpoint != "https://provider.example.invalid/v1" || selected . Target . Model != "inherited-model" || selected . Target . Temperature != 0 || selected . Target . APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected . Target . ExtraParams != nil {
t . Fatalf ( "resolved inherited profile = %d %q %q %#v" , code , output , diagnostic , selected )
}
code , output , diagnostic , inherited := run ( "inherited" )
if code != 0 || diagnostic != "" || inherited . Target . ExtraParams [ "Authorization" ] != "[REDACTED]" || inherited . Target . ExtraParams [ "ordinary" ] != "visible-value" || strings . Contains ( output , "private-authorization" ) {
t . Fatalf ( "redacted profile = %d %q %q %#v" , code , output , diagnostic , inherited )
}
code , output , diagnostic , endpointOnly := run ( "endpoint-only" )
if code != 0 || diagnostic != "" || endpointOnly . ProfileID != "endpoint-only" || endpointOnly . APIKeyRequired || endpointOnly . Target . BackendID != "" || endpointOnly . Target . Endpoint != "https://endpoint.example.invalid/v1" {
t . Fatalf ( "endpoint-only profile = %d %q %q %#v" , code , output , diagnostic , endpointOnly )
}
code , output , diagnostic , literal := run ( " literal profile " )
if code != 0 || diagnostic != "" || literal . ProfileID != " literal profile " || literal . Target . Model != "literal-model" {
t . Fatalf ( "literal profile ID = %d %q %q %#v" , code , output , diagnostic , literal )
}
if lookupCalls != 0 || factoryCalls != 0 {
t . Fatalf ( "profile inspection called credential lookup %d times and LLM factory %d times" , lookupCalls , factoryCalls )
}
}
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures ( t * testing . T ) {
t . Parallel ( )
configPath := writeCommandConfig ( t , "session" , "other" )
for _ , test := range [ ] struct {
name string
args [ ] string
want int
} {
{ name : "missing ID" , args : [ ] string { "config" , "profile" , "--config" , configPath } , want : 2 } ,
{ name : "blank ID" , args : [ ] string { "config" , "profile" , "--config" , configPath , " " } , want : 2 } ,
{ name : "multiple IDs" , args : [ ] string { "config" , "profile" , "--config" , configPath , "one" , "two" } , want : 2 } ,
{ name : "unknown profile" , args : [ ] string { "config" , "profile" , "--config" , configPath , "unknown-profile" } , want : 1 } ,
} {
t . Run ( test . name , func ( t * testing . T ) {
var stdout , stderr bytes . Buffer
if code := RunWithOptions ( test . args , & stdout , & stderr , commandContractOptions ( t ) ) ; code != test . want || stderr . Len ( ) == 0 {
t . Fatalf ( "code=%d stdout=%q stderr=%q, want exit %d and diagnostic" , code , stdout . String ( ) , stderr . String ( ) , test . want )
}
} )
}
profilePath := filepath . Join ( t . TempDir ( ) , "profiles.yml" )
writeConfigKitFile ( t , profilePath , "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n" )
configPath = writeCommandConfigContent ( t , "version: 5\npromptkit:\n profile_file: " + profilePath + "\n" )
var stderr bytes . Buffer
if code := RunWithOptions ( [ ] string { "config" , "profile" , "--config" , configPath , "writer" } , failingInspectionWriter { } , & stderr , commandContractOptions ( t ) ) ; code != 1 || ! strings . Contains ( stderr . String ( ) , "write PromptKit profile inspection" ) {
t . Fatalf ( "profile writer failure = %d, %q" , code , stderr . String ( ) )
}
}
2026-09-13 17:16:06 +00:00
type failingInspectionWriter struct { }
func ( failingInspectionWriter ) Write ( [ ] byte ) ( int , error ) {
return 0 , io . ErrClosedPipe
}