Migrate configuration and command tests to native inspection
This commit is contained in:
@@ -6,48 +6,27 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/buildinfo"
|
||||
)
|
||||
|
||||
func TestCommandHelpSpellingsWriteUsageToStdout(t *testing.T) {
|
||||
tests := [][]string{nil, {"help"}, {"--help"}, {"-h"}}
|
||||
for _, args := range tests {
|
||||
name := "no arguments"
|
||||
if len(args) > 0 {
|
||||
name = args[0]
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"unknown"},
|
||||
{"pipeline", "unknown"},
|
||||
{"config", "validate", "--unknown"},
|
||||
{"run", "demo", "--input-file", "missing", "--cache", "invalid"},
|
||||
{"pipeline", "validate"},
|
||||
{"run", "demo", "--input-file", "missing", "--only", "spells"},
|
||||
} {
|
||||
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 0 || !strings.Contains(stdout.String(), "Usage:") || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{name: "unknown command", args: []string{"unknown"}, want: "unknown command"},
|
||||
{name: "missing config subcommand", args: []string{"config"}, want: "config requires a subcommand"},
|
||||
{name: "unknown pipelines subcommand", args: []string{"pipelines", "unknown"}, want: "unknown pipelines subcommand"},
|
||||
{name: "malformed run flag", args: []string{"run", "demo", "--chunk_cache", "invalid"}, want: "not supported"},
|
||||
{name: "unknown flag", args: []string{"config", "validate", "--unknown"}, want: "flag provided but not defined"},
|
||||
{name: "version arguments", args: []string{"--version", "extra"}, want: "--version does not accept arguments"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions(tt.args, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 2 || !strings.Contains(stderr.String(), tt.want) || stdout.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -56,25 +35,21 @@ func TestCommandSyntaxErrorsUseStderrAndExitTwo(t *testing.T) {
|
||||
func TestCommandVersionOutput(t *testing.T) {
|
||||
previous := buildinfo.Override
|
||||
t.Cleanup(func() { buildinfo.Override = previous })
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
override string
|
||||
wantCode int
|
||||
wantStdout string
|
||||
wantStderr string
|
||||
for _, test := range []struct {
|
||||
name, override string
|
||||
status int
|
||||
fragment string
|
||||
}{
|
||||
{name: "development", wantStdout: "notarius development\n"},
|
||||
{name: "release override", override: "v1.2.3", wantStdout: "notarius v1.2.3\n"},
|
||||
{name: "invalid override", override: "release", wantCode: 1, wantStderr: "notarius: build version override is not a stable release tag\n"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
buildinfo.Override = tt.override
|
||||
{name: "development", fragment: "development"},
|
||||
{name: "release", override: "v1.2.3", fragment: "v1.2.3"},
|
||||
{name: "invalid release identity", override: "release", status: 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
buildinfo.Override = test.override
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"--version"}, &stdout, &stderr, Options{})
|
||||
if code != tt.wantCode || stdout.String() != tt.wantStdout || stderr.String() != tt.wantStderr {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
code := RunWithOptions([]string{"version"}, &stdout, &stderr, Options{})
|
||||
if code != test.status || test.status == 0 && (!strings.Contains(stdout.String(), test.fragment) || stderr.Len() != 0) || test.status != 0 && (stdout.Len() != 0 || stderr.Len() == 0) {
|
||||
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -83,143 +58,104 @@ func TestCommandVersionOutput(t *testing.T) {
|
||||
func TestConfigDiscoveryPrefersExplicitPathThenEnvironment(t *testing.T) {
|
||||
explicit := writeCommandConfig(t, "explicit", "alpha")
|
||||
environment := writeCommandConfig(t, "environment", "beta")
|
||||
lookup := func(name string) (string, bool) {
|
||||
if name == "NOTARIUS_CONFIG" {
|
||||
return environment, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"pipelines", "list", "--config", explicit}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
|
||||
if code != 0 || stdout.String() != "alpha\nexplicit\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("explicit config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"pipelines", "list"}, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
|
||||
if code != 0 || stdout.String() != "beta\nenvironment\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("environment config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
lookup := func(name string) (string, bool) { return environment, name == "NOTARIUS_CONFIG" }
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
flags []string
|
||||
want []string
|
||||
}{
|
||||
{name: "explicit", flags: []string{"--config", explicit}, want: []string{"alpha", "explicit"}},
|
||||
{name: "environment", want: []string{"beta", "environment"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
args := append([]string{"pipeline", "list", "--format", "json"}, test.flags...)
|
||||
code := RunWithOptions(args, &stdout, &stderr, commandContractOptionsWithLookup(t, lookup))
|
||||
var ids []string
|
||||
if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code != 0 || !reflect.DeepEqual(ids, test.want) {
|
||||
t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigLoadingFailuresReturnOneWithPathContext(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "missing.yml")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", missing}, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 1 || !strings.Contains(stderr.String(), missing) || stdout.Len() != 0 {
|
||||
t.Fatalf("missing config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
malformed := filepath.Join(t.TempDir(), "malformed.yml")
|
||||
if err := os.WriteFile(malformed, []byte("version: [\n"), 0o600); err != nil {
|
||||
func TestConfigLoadingFailuresRetainPathContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
missing := filepath.Join(root, "missing.yml")
|
||||
malformed := filepath.Join(root, "malformed.yml")
|
||||
if err := os.WriteFile(malformed, []byte("version: [\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", malformed}, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != 1 || !strings.Contains(stderr.String(), "configuration contains invalid YAML") || stdout.Len() != 0 {
|
||||
t.Fatalf("malformed config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
for _, path := range []string{missing, malformed} {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", path}, &stdout, &stderr, commandContractOptions(t))
|
||||
if code == 0 || !strings.Contains(stderr.String(), path) {
|
||||
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidateResolvesPipelineAndChecksSelection(t *testing.T) {
|
||||
configPath := writeResolvableCommandConfig(t)
|
||||
options := commandContractOptions(t)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells"}, &stdout, &stderr, options)
|
||||
if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 {
|
||||
t.Fatalf("valid resolution: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "missing"}, &stdout, &stderr, options)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") {
|
||||
t.Fatalf("unknown pipeline: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "missing"}, &stdout, &stderr, options)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "lane \"missing\"") {
|
||||
t.Fatalf("unknown lane: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--only", "spells"}, &stdout, &stderr, options)
|
||||
if code != 2 || !strings.Contains(stderr.String(), "--only requires --pipeline") {
|
||||
t.Fatalf("missing pipeline for only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo", "--only", "spells,,other"}, &stdout, &stderr, options)
|
||||
if code != 2 || !strings.Contains(stderr.String(), "--only must contain") {
|
||||
t.Fatalf("malformed only: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
func TestPipelineValidationSelectsConfiguredCompositionOffline(t *testing.T) {
|
||||
path := writeResolvableCommandConfig(t)
|
||||
for _, test := range []struct {
|
||||
id string
|
||||
status int
|
||||
}{
|
||||
{id: "demo"}, {id: "missing", status: 2},
|
||||
} {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"pipeline", "validate", test.id, "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t))
|
||||
if code != test.status {
|
||||
t.Fatalf("status=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if code == 0 {
|
||||
var report struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Valid bool `json:"valid"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Pipeline != test.id || !report.Valid {
|
||||
t.Fatalf("validation report=%#v", report)
|
||||
}
|
||||
} else if !strings.Contains(stderr.String(), test.id) {
|
||||
t.Fatalf("missing pipeline context: %s", stderr.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigValidatePipelineDefaultProfileIsOffline(t *testing.T) {
|
||||
configPath := writeCommandConfigContent(t, `version: 5
|
||||
func TestPipelineValidationDefaultProfileNeedsNoCredentialsOrProvider(t *testing.T) {
|
||||
path := writeCommandConfigContent(t, `version: 6
|
||||
pipelines:
|
||||
demo:
|
||||
llm_profile: dnd-extraction
|
||||
input:
|
||||
module: seriatim
|
||||
llm_profile: notarius/dnd-extraction
|
||||
input: {module: seriatim}
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
extract: {module: dnd/spells}
|
||||
`)
|
||||
client := &diagnosticClient{}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "demo"}, &stdout, &stderr, Options{})
|
||||
if code != 0 || !strings.Contains(stdout.String(), "valid for pipeline \"demo\"") || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
code := RunWithOptions([]string{"pipeline", "validate", "demo", "--config", path}, &stdout, &stderr, Options{LLMClient: client, LookupEnv: emptyLookup})
|
||||
if code != 0 || client.calls.Load() != 0 {
|
||||
t.Fatalf("status=%d calls=%d stderr=%q", code, client.calls.Load(), stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelinesListSortsNormalizedIDsInTextAndJSON(t *testing.T) {
|
||||
configPath := writeCommandConfig(t, " zeta ", "alpha")
|
||||
options := commandContractOptions(t)
|
||||
func TestPipelineListSortsNormalizedIDs(t *testing.T) {
|
||||
path := writeCommandConfig(t, " zeta ", "alpha")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"pipelines", "list", "--config", configPath}, &stdout, &stderr, options)
|
||||
if code != 0 || stdout.String() != "alpha\nzeta\n" || stderr.Len() != 0 {
|
||||
t.Fatalf("text list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
code := RunWithOptions([]string{"pipeline", "list", "--config", path, "--format", "json"}, &stdout, &stderr, commandContractOptions(t))
|
||||
var ids []string
|
||||
if err := json.Unmarshal(stdout.Bytes(), &ids); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"pipelines", "list", "--config", configPath, "--json"}, &stdout, &stderr, options)
|
||||
var payload struct {
|
||||
Pipelines []string `json:"pipelines"`
|
||||
}
|
||||
if err := json.Unmarshal(stdout.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("JSON list = %q: %v", stdout.String(), err)
|
||||
}
|
||||
if code != 0 || len(payload.Pipelines) != 2 || payload.Pipelines[0] != "alpha" || payload.Pipelines[1] != "zeta" || stderr.Len() != 0 {
|
||||
t.Fatalf("JSON list: code=%d payload=%#v stderr=%q", code, payload, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovedStructuralFlagsAndRuntimeFailuresKeepExitClasses(t *testing.T) {
|
||||
configPath := writeResolvableCommandConfig(t)
|
||||
options := commandContractOptions(t)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "demo", "--input", "missing-input", "--config", configPath, "--diagnostics-dir", t.TempDir()}, &stdout, &stderr, options)
|
||||
if code != 2 || !strings.Contains(stderr.String(), "flag provided but not defined") {
|
||||
t.Fatalf("removed flag: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"run", "missing", "--input", "missing-input", "--config", configPath, "--chunk_cache", "bypass"}, &stdout, &stderr, options)
|
||||
if code != 1 || !strings.Contains(stderr.String(), "pipeline \"missing\"") || stdout.Len() != 0 {
|
||||
t.Fatalf("valid-runtime failure: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
if code != 0 || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) {
|
||||
t.Fatalf("status=%d ids=%#v stderr=%q", code, ids, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,42 +165,29 @@ func commandContractOptions(t *testing.T) Options {
|
||||
|
||||
func commandContractOptionsWithLookup(t *testing.T, lookup func(string) (string, bool)) Options {
|
||||
t.Helper()
|
||||
components, err := newProductionComponents()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Options{
|
||||
Catalog: catalogFromRegistries(components.registries),
|
||||
Registries: components.registries,
|
||||
LookupEnv: lookup,
|
||||
}
|
||||
options := productionCLIOptions(t)
|
||||
options.LookupEnv = lookup
|
||||
return options
|
||||
}
|
||||
|
||||
func writeCommandConfig(t *testing.T, firstID, secondID string) string {
|
||||
t.Helper()
|
||||
content := fmt.Sprintf("version: 5\npipelines:\n %q:\n input:\n module: seriatim\n %q:\n input:\n module: seriatim\n", firstID, secondID)
|
||||
return writeCommandConfigContent(t, content)
|
||||
return writeCommandConfigContent(t, fmt.Sprintf("version: 6\npipelines:\n %q:\n input: {module: seriatim}\n %q:\n input: {module: seriatim}\n", firstID, secondID))
|
||||
}
|
||||
|
||||
func writeResolvableCommandConfig(t *testing.T) string {
|
||||
t.Helper()
|
||||
return writeCommandConfigContent(t, `version: 5
|
||||
return writeCommandConfigContent(t, `version: 6
|
||||
pipelines:
|
||||
demo:
|
||||
input:
|
||||
module: seriatim
|
||||
input: {module: seriatim}
|
||||
artifacts:
|
||||
spells:
|
||||
extract:
|
||||
module: dnd/spells
|
||||
extract: {module: dnd/spells}
|
||||
`)
|
||||
}
|
||||
|
||||
func writeCommandConfigContent(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
return writeProductionContractConfig(t, content)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user