525 lines
23 KiB
Go
525 lines
23 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func TestExplainRejectsMalformedPointerBeforeLoadingEitherScope(t *testing.T) {
|
|
for _, scope := range []string{"notarius", "promptkit"} {
|
|
t.Run(scope, func(t *testing.T) {
|
|
var stdout, stderr strings.Builder
|
|
options := commandContractOptions(t)
|
|
options.LookupEnv = func(string) (string, bool) {
|
|
t.Fatal("configuration discovery ran for invalid syntax")
|
|
return "", false
|
|
}
|
|
code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", filepath.Join(t.TempDir(), "missing.yml"), "/profiles/~2"}, &stdout, &stderr, options)
|
|
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "canonical JSON Pointer") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
|
|
writeConfigKitFile(t, path, "version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: input}\n")
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
|
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") {
|
|
t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
for _, selection := range [][]string{nil, {"--config-overlay", ""}, {"--config-overlay", "default"}} {
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
args := append([]string{"pipelines", "list", "--config", path}, selection...)
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\n" || stderr.Len() != 0 {
|
|
t.Fatalf("base list %q = %d, %q, %q", selection, code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfigContent(t, `version: 5
|
|
pipelines:
|
|
" parent ":
|
|
input:
|
|
module: seriatim
|
|
artifacts:
|
|
spells:
|
|
extract:
|
|
module: dnd/spells
|
|
structured_output_repair_attempts: 2
|
|
child:
|
|
extends: " parent "
|
|
structured_output_repair_attempts: 0
|
|
`)
|
|
options := commandContractOptions(t)
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 {
|
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 {
|
|
t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 {
|
|
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 {
|
|
t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) {
|
|
t.Parallel()
|
|
for _, test := range []struct {
|
|
name string
|
|
content string
|
|
args []string
|
|
want string
|
|
}{
|
|
{
|
|
name: "missing parent",
|
|
content: `version: 5
|
|
pipelines:
|
|
child:
|
|
extends: absent
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "inheritance.missing",
|
|
},
|
|
{
|
|
name: "cycle",
|
|
content: `version: 5
|
|
pipelines:
|
|
first:
|
|
extends: second
|
|
second:
|
|
extends: first
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "first"},
|
|
want: "cycle",
|
|
},
|
|
{
|
|
name: "invalid directive type",
|
|
content: `version: 5
|
|
pipelines:
|
|
child:
|
|
extends: 7
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "extends",
|
|
},
|
|
{
|
|
name: "unselected missing parent",
|
|
content: `version: 5
|
|
pipelines:
|
|
parent:
|
|
input:
|
|
module: seriatim
|
|
child:
|
|
extends: parent
|
|
unrelated:
|
|
extends: absent
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "inheritance.missing",
|
|
},
|
|
{
|
|
name: "exact sibling mismatch",
|
|
content: `version: 5
|
|
pipelines:
|
|
parent:
|
|
input:
|
|
module: seriatim
|
|
child:
|
|
extends: "parent "
|
|
`,
|
|
args: []string{"pipelines", "list"},
|
|
want: "inheritance.missing",
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfigContent(t, test.content)
|
|
args := append([]string{}, test.args...)
|
|
args = append(args, "--config", path)
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, args := range [][]string{
|
|
{"run", "sample", "--config-profile=development"},
|
|
{"config", "validate", "--config-profile=development"},
|
|
{"config", "effective", "--config-profile=development"},
|
|
{"config", "explain", "--config-profile=development", ""},
|
|
{"pipelines", "list", "--config-profile=development"},
|
|
} {
|
|
t.Run(strings.Join(args[:2], " "), func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "not defined") {
|
|
t.Fatalf("args=%q code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) {
|
|
t.Parallel()
|
|
path := filepath.Join(t.TempDir(), "notarius.yml")
|
|
writeConfigKitFile(t, path, "version: 4\n")
|
|
var stdout, stderr bytes.Buffer
|
|
if code := runPipelinesList([]string{"--config", path}, &stdout, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "version") {
|
|
t.Fatalf("list = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionCommandsRenderSafeConfigKitViews(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, path, `version: 5
|
|
concurrency:
|
|
total_llm: 4
|
|
pipelines:
|
|
"0":
|
|
input: {module: input}
|
|
`)
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency:\n total_llm: 3\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "retired.yml"), "pipelines: null\n")
|
|
|
|
providerCalled := false
|
|
opts := commandContractOptions(t)
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
providerCalled = true
|
|
return nil, nil, errors.New("LLM client must not be constructed")
|
|
}
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 {
|
|
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
if providerCalled {
|
|
t.Fatal("config effective constructed an LLM client")
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, ""}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: ""`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain root = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestConfigExplainRejectsInvalidAndUnknownPointers(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
for _, test := range []struct {
|
|
name string
|
|
pointer string
|
|
wantCode int
|
|
}{
|
|
{name: "missing", wantCode: 2},
|
|
{name: "malformed escape", pointer: "/pipelines~2session", wantCode: 2},
|
|
{name: "unknown", pointer: "/unknown", wantCode: 1},
|
|
{name: "invalid sequence index", pointer: "/pipelines/session/steps/+0", wantCode: 1},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
args := []string{"config", "explain", "--config", path}
|
|
if test.name != "missing" {
|
|
args = append(args, test.pointer)
|
|
}
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != test.wantCode || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
var stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
|
t.Fatalf("effective writer failure = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
configPath := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
|
|
base:
|
|
endpoint: https://example.invalid/v1
|
|
model: base-model
|
|
extra_params:
|
|
Authorization: base-auth-secret
|
|
ordinary: visible-value
|
|
`)
|
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
|
|
imports:
|
|
- path: base.yml
|
|
profiles:
|
|
"escaped/id":
|
|
model: imported-model
|
|
dnd-extraction:
|
|
model: imported-override
|
|
`)
|
|
assets, err := productionPromptAssets()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
factoryCalls := 0
|
|
opts := commandContractOptions(t)
|
|
opts.promptKitAssets = assets
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
factoryCalls++
|
|
return nil, nil, errors.New("inspection must not construct an LLM client")
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
|
|
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
defaultReport := stdout.String()
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
|
|
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
if factoryCalls != 0 {
|
|
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
opts := commandContractOptions(t)
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
want int
|
|
}{
|
|
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
|
|
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
|
|
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
|
|
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
|
}
|
|
})
|
|
}
|
|
var stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
|
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
profilePath := filepath.Join(root, "profiles.yml")
|
|
writeConfigKitFile(t, profilePath, `backends:
|
|
managed:
|
|
endpoint: https://provider.example.invalid/v1
|
|
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
|
profiles:
|
|
inherited:
|
|
backend: managed
|
|
model: inherited-model
|
|
temperature: 0.4
|
|
api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY
|
|
api_key_required: true
|
|
extra_params:
|
|
Authorization: private-authorization
|
|
ordinary: visible-value
|
|
selected:
|
|
extends: inherited
|
|
temperature: 0
|
|
extra_params: null
|
|
endpoint-only:
|
|
endpoint: https://endpoint.example.invalid/v1
|
|
model: endpoint-model
|
|
api_key_required: false
|
|
" literal profile ":
|
|
endpoint: https://literal.example.invalid/v1
|
|
model: literal-model
|
|
`)
|
|
configPath := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n")
|
|
assets, err := productionPromptAssets()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lookupCalls := 0
|
|
factoryCalls := 0
|
|
opts := commandContractOptions(t)
|
|
opts.promptKitAssets = assets
|
|
opts.LookupEnv = func(string) (string, bool) {
|
|
lookupCalls++
|
|
return "", false
|
|
}
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
factoryCalls++
|
|
return nil, nil, errors.New("profile inspection must not construct an LLM client")
|
|
}
|
|
|
|
type profileReport struct {
|
|
ProfileID string `json:"profile_id"`
|
|
APIKeyRequired bool `json:"api_key_required"`
|
|
Target struct {
|
|
BackendID string `json:"backend_id"`
|
|
Endpoint string `json:"endpoint"`
|
|
Model string `json:"model"`
|
|
Temperature float64 `json:"temperature"`
|
|
APIKeyEnv string `json:"api_key_env"`
|
|
ExtraParams map[string]any `json:"extra_params"`
|
|
} `json:"effective_model_params"`
|
|
}
|
|
run := func(profileID string) (int, string, string, profileReport) {
|
|
t.Helper()
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts)
|
|
var report profileReport
|
|
if code == 0 {
|
|
if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 {
|
|
t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String())
|
|
}
|
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
|
t.Fatalf("decode profile JSON %q: %v", stdout.String(), err)
|
|
}
|
|
}
|
|
return code, stdout.String(), stderr.String(), report
|
|
}
|
|
code, output, diagnostic, selected := run("selected")
|
|
if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil {
|
|
t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected)
|
|
}
|
|
code, output, diagnostic, inherited := run("inherited")
|
|
if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") {
|
|
t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited)
|
|
}
|
|
code, output, diagnostic, endpointOnly := run("endpoint-only")
|
|
if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" {
|
|
t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly)
|
|
}
|
|
code, output, diagnostic, literal := run(" literal profile ")
|
|
if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" {
|
|
t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal)
|
|
}
|
|
if lookupCalls != 0 || factoryCalls != 0 {
|
|
t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls)
|
|
}
|
|
}
|
|
|
|
func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) {
|
|
t.Parallel()
|
|
configPath := writeCommandConfig(t, "session", "other")
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
want int
|
|
}{
|
|
{name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2},
|
|
{name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2},
|
|
{name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2},
|
|
{name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
|
}
|
|
})
|
|
}
|
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
|
writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n")
|
|
configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n")
|
|
var stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") {
|
|
t.Fatalf("profile writer failure = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
type failingInspectionWriter struct{}
|
|
|
|
func (failingInspectionWriter) Write([]byte) (int, error) {
|
|
return 0, io.ErrClosedPipe
|
|
}
|