380 lines
16 KiB
Go
380 lines
16 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
|
|
writeConfigKitFile(t, path, "version: 5\nconfigkit:\n imports:\n - path: fragment.yml\npipelines:\n base:\n input: {module: input}\n")
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
|
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") {
|
|
t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
for _, selection := range [][]string{nil, {"--config-overlay", ""}, {"--config-overlay", "default"}} {
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
args := append([]string{"pipelines", "list", "--config", path}, selection...)
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\n" || stderr.Len() != 0 {
|
|
t.Fatalf("base list %q = %d, %q, %q", selection, code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfigContent(t, `version: 5
|
|
pipelines:
|
|
" parent ":
|
|
input:
|
|
module: seriatim
|
|
artifacts:
|
|
spells:
|
|
extract:
|
|
module: dnd/spells
|
|
structured_output_repair_attempts: 2
|
|
child:
|
|
extends: " parent "
|
|
structured_output_repair_attempts: 0
|
|
`)
|
|
options := commandContractOptions(t)
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 {
|
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 {
|
|
t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 {
|
|
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 {
|
|
t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) {
|
|
t.Parallel()
|
|
for _, test := range []struct {
|
|
name string
|
|
content string
|
|
args []string
|
|
want string
|
|
}{
|
|
{
|
|
name: "missing parent",
|
|
content: `version: 5
|
|
pipelines:
|
|
child:
|
|
extends: absent
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "inheritance.missing",
|
|
},
|
|
{
|
|
name: "cycle",
|
|
content: `version: 5
|
|
pipelines:
|
|
first:
|
|
extends: second
|
|
second:
|
|
extends: first
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "first"},
|
|
want: "cycle",
|
|
},
|
|
{
|
|
name: "invalid directive type",
|
|
content: `version: 5
|
|
pipelines:
|
|
child:
|
|
extends: 7
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "extends",
|
|
},
|
|
{
|
|
name: "unselected missing parent",
|
|
content: `version: 5
|
|
pipelines:
|
|
parent:
|
|
input:
|
|
module: seriatim
|
|
child:
|
|
extends: parent
|
|
unrelated:
|
|
extends: absent
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "inheritance.missing",
|
|
},
|
|
{
|
|
name: "exact sibling mismatch",
|
|
content: `version: 5
|
|
pipelines:
|
|
parent:
|
|
input:
|
|
module: seriatim
|
|
child:
|
|
extends: "parent "
|
|
`,
|
|
args: []string{"pipelines", "list"},
|
|
want: "inheritance.missing",
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfigContent(t, test.content)
|
|
args := append([]string{}, test.args...)
|
|
args = append(args, "--config", path)
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, args := range [][]string{
|
|
{"run", "sample", "--config-profile=development"},
|
|
{"config", "validate", "--config-profile=development"},
|
|
{"config", "effective", "--config-profile=development"},
|
|
{"config", "explain", "--config-profile=development", ""},
|
|
{"pipelines", "list", "--config-profile=development"},
|
|
} {
|
|
t.Run(strings.Join(args[:2], " "), func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "not defined") {
|
|
t.Fatalf("args=%q code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) {
|
|
t.Parallel()
|
|
path := filepath.Join(t.TempDir(), "notarius.yml")
|
|
writeConfigKitFile(t, path, "version: 4\n")
|
|
var stdout, stderr bytes.Buffer
|
|
if code := runPipelinesList([]string{"--config", path}, &stdout, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "version") {
|
|
t.Fatalf("list = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionCommandsRenderSafeConfigKitViews(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, path, `version: 5
|
|
concurrency:
|
|
total_llm: 4
|
|
pipelines:
|
|
"0":
|
|
input: {module: input}
|
|
`)
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency:\n total_llm: 3\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "retired.yml"), "pipelines: null\n")
|
|
|
|
providerCalled := false
|
|
opts := commandContractOptions(t)
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
providerCalled = true
|
|
return nil, nil, errors.New("LLM client must not be constructed")
|
|
}
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 {
|
|
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
if providerCalled {
|
|
t.Fatal("config effective constructed an LLM client")
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, ""}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: ""`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain root = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestConfigExplainRejectsInvalidAndUnknownPointers(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
for _, test := range []struct {
|
|
name string
|
|
pointer string
|
|
wantCode int
|
|
}{
|
|
{name: "missing", wantCode: 2},
|
|
{name: "malformed escape", pointer: "/pipelines~2session", wantCode: 2},
|
|
{name: "unknown", pointer: "/unknown", wantCode: 1},
|
|
{name: "invalid sequence index", pointer: "/pipelines/session/steps/+0", wantCode: 1},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
args := []string{"config", "explain", "--config", path}
|
|
if test.name != "missing" {
|
|
args = append(args, test.pointer)
|
|
}
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != test.wantCode || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
var stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
|
t.Fatalf("effective writer failure = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
configPath := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
|
|
base:
|
|
endpoint: https://example.invalid/v1
|
|
model: base-model
|
|
extra_params:
|
|
Authorization: base-auth-secret
|
|
ordinary: visible-value
|
|
`)
|
|
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
|
|
imports:
|
|
- path: base.yml
|
|
profiles:
|
|
"escaped/id":
|
|
model: imported-model
|
|
dnd-extraction:
|
|
model: imported-override
|
|
`)
|
|
assets, err := productionPromptAssets()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
factoryCalls := 0
|
|
opts := commandContractOptions(t)
|
|
opts.promptKitAssets = assets
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
factoryCalls++
|
|
return nil, nil, errors.New("inspection must not construct an LLM client")
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
|
|
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
defaultReport := stdout.String()
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
|
|
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
|
|
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
if factoryCalls != 0 {
|
|
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
opts := commandContractOptions(t)
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
want int
|
|
}{
|
|
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
|
|
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
|
|
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
|
|
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
|
}
|
|
})
|
|
}
|
|
var stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
|
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
type failingInspectionWriter struct{}
|
|
|
|
func (failingInspectionWriter) Write([]byte) (int, error) {
|
|
return 0, io.ErrClosedPipe
|
|
}
|