package cli import ( "bytes" "context" "encoding/json" "errors" "io" "path/filepath" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) func TestExplainRejectsMalformedPointerBeforeLoadingEitherScope(t *testing.T) { for _, scope := range []string{"notarius", "promptkit"} { t.Run(scope, func(t *testing.T) { var stdout, stderr strings.Builder options := commandContractOptions(t) options.LookupEnv = func(string) (string, bool) { t.Fatal("configuration discovery ran for invalid syntax") return "", false } code := RunWithOptions([]string{"config", "explain", "--scope", scope, "--config", filepath.Join(t.TempDir(), "missing.yml"), "/profiles/~2"}, &stdout, &stderr, options) if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "canonical JSON Pointer") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } } func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) { t.Parallel() root := t.TempDir() path := filepath.Join(root, "notarius.yml") writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n") writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n") writeConfigKitFile(t, path, "version: 5\nconfig:\n imports:\n - path: fragment.yml\n required: true\npipelines:\n base:\n input: {module: input}\n") var stdout, stderr bytes.Buffer if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" { t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") { t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String()) } for _, selection := range [][]string{nil, {"--config-overlay", ""}, {"--config-overlay", "default"}} { stdout.Reset() stderr.Reset() args := append([]string{"pipelines", "list", "--config", path}, selection...) if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\n" || stderr.Len() != 0 { t.Fatalf("base list %q = %d, %q, %q", selection, code, stdout.String(), stderr.String()) } } } func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) { t.Parallel() path := writeCommandConfigContent(t, `version: 5 pipelines: " parent ": input: module: seriatim artifacts: spells: extract: module: dnd/spells structured_output_repair_attempts: 2 child: extends: " parent " structured_output_repair_attempts: 0 `) options := commandContractOptions(t) var stdout, stderr bytes.Buffer if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 { t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 { t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 { t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 { t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String()) } } func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) { t.Parallel() for _, test := range []struct { name string content string args []string want string }{ { name: "missing parent", content: `version: 5 pipelines: child: extends: absent `, args: []string{"config", "validate", "--pipeline", "child"}, want: "inheritance.missing", }, { name: "cycle", content: `version: 5 pipelines: first: extends: second second: extends: first `, args: []string{"config", "validate", "--pipeline", "first"}, want: "cycle", }, { name: "invalid directive type", content: `version: 5 pipelines: child: extends: 7 `, args: []string{"config", "validate", "--pipeline", "child"}, want: "extends", }, { name: "unselected missing parent", content: `version: 5 pipelines: parent: input: module: seriatim child: extends: parent unrelated: extends: absent `, args: []string{"config", "validate", "--pipeline", "child"}, want: "inheritance.missing", }, { name: "exact sibling mismatch", content: `version: 5 pipelines: parent: input: module: seriatim child: extends: "parent " `, args: []string{"pipelines", "list"}, want: "inheritance.missing", }, } { t.Run(test.name, func(t *testing.T) { t.Parallel() path := writeCommandConfigContent(t, test.content) args := append([]string{}, test.args...) args = append(args, "--config", path) var stdout, stderr bytes.Buffer if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } } func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) { t.Parallel() for _, args := range [][]string{ {"run", "sample", "--config-profile=development"}, {"config", "validate", "--config-profile=development"}, {"config", "effective", "--config-profile=development"}, {"config", "explain", "--config-profile=development", ""}, {"pipelines", "list", "--config-profile=development"}, } { t.Run(strings.Join(args[:2], " "), func(t *testing.T) { var stdout, stderr bytes.Buffer if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "not defined") { t.Fatalf("args=%q code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) } }) } } func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "notarius.yml") writeConfigKitFile(t, path, "version: 4\n") var stdout, stderr bytes.Buffer if code := runPipelinesList([]string{"--config", path}, &stdout, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "version") { t.Fatalf("list = %d, %q", code, stderr.String()) } } func TestConfigurationInspectionCommandsRenderSafeConfigKitViews(t *testing.T) { t.Parallel() root := t.TempDir() path := filepath.Join(root, "notarius.yml") writeConfigKitFile(t, path, `version: 5 concurrency: total_llm: 4 pipelines: "0": input: {module: input} `) writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency:\n total_llm: 3\n") writeConfigKitFile(t, filepath.Join(root, "overlays", "retired.yml"), "pipelines: null\n") providerCalled := false opts := commandContractOptions(t) opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { providerCalled = true return nil, nil, errors.New("LLM client must not be constructed") } var stdout, stderr bytes.Buffer if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 { t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String()) } if providerCalled { t.Fatal("config effective constructed an LLM client") } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 { t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "explain", "--config", path, ""}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: ""`) || stderr.Len() != 0 { t.Fatalf("explain root = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 { t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String()) } } func TestConfigExplainRejectsInvalidAndUnknownPointers(t *testing.T) { t.Parallel() path := writeCommandConfig(t, "session", "other") for _, test := range []struct { name string pointer string wantCode int }{ {name: "missing", wantCode: 2}, {name: "malformed escape", pointer: "/pipelines~2session", wantCode: 2}, {name: "unknown", pointer: "/unknown", wantCode: 1}, {name: "invalid sequence index", pointer: "/pipelines/session/steps/+0", wantCode: 1}, } { t.Run(test.name, func(t *testing.T) { var stdout, stderr bytes.Buffer args := []string{"config", "explain", "--config", path} if test.name != "missing" { args = append(args, test.pointer) } if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != test.wantCode || stdout.Len() != 0 || stderr.Len() == 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } } func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) { t.Parallel() path := writeCommandConfig(t, "session", "other") var stderr bytes.Buffer if code := RunWithOptions([]string{"config", "effective", "--config", path}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") { t.Fatalf("effective writer failure = %d, %q", code, stderr.String()) } } func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) { t.Parallel() root := t.TempDir() configPath := filepath.Join(root, "notarius.yml") writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n") writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n") writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles: base: endpoint: https://example.invalid/v1 model: base-model extra_params: Authorization: base-auth-secret ordinary: visible-value `) writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit: imports: - path: base.yml profiles: "escaped/id": model: imported-model dnd-extraction: model: imported-override `) assets, err := productionPromptAssets() if err != nil { t.Fatal(err) } factoryCalls := 0 opts := commandContractOptions(t) opts.promptKitAssets = assets opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { factoryCalls++ return nil, nil, errors.New("inspection must not construct an LLM client") } var stdout, stderr bytes.Buffer if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 { t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String()) } defaultReport := stdout.String() stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 { t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 { t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 { t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 { t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String()) } stdout.Reset() stderr.Reset() if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 { t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String()) } if factoryCalls != 0 { t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls) } } func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) { t.Parallel() path := writeCommandConfig(t, "session", "other") opts := commandContractOptions(t) for _, test := range []struct { name string args []string want int }{ {name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2}, {name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2}, {name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2}, {name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1}, } { t.Run(test.name, func(t *testing.T) { var stdout, stderr bytes.Buffer if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 { t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want) } }) } var stderr bytes.Buffer if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") { t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String()) } } func TestConfigProfileReportsResolvedRedactedSettingsWithoutCredentialLookup(t *testing.T) { t.Parallel() root := t.TempDir() profilePath := filepath.Join(root, "profiles.yml") writeConfigKitFile(t, profilePath, `backends: managed: endpoint: https://provider.example.invalid/v1 api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY profiles: inherited: backend: managed model: inherited-model temperature: 0.4 api_key_env: NOT_SET_SYNTHETIC_PROFILE_KEY api_key_required: true extra_params: Authorization: private-authorization ordinary: visible-value selected: extends: inherited temperature: 0 extra_params: null endpoint-only: endpoint: https://endpoint.example.invalid/v1 model: endpoint-model api_key_required: false " literal profile ": endpoint: https://literal.example.invalid/v1 model: literal-model `) configPath := filepath.Join(root, "notarius.yml") writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n") writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+profilePath+"\n") assets, err := productionPromptAssets() if err != nil { t.Fatal(err) } lookupCalls := 0 factoryCalls := 0 opts := commandContractOptions(t) opts.promptKitAssets = assets opts.LookupEnv = func(string) (string, bool) { lookupCalls++ return "", false } opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { factoryCalls++ return nil, nil, errors.New("profile inspection must not construct an LLM client") } type profileReport struct { ProfileID string `json:"profile_id"` APIKeyRequired bool `json:"api_key_required"` Target struct { BackendID string `json:"backend_id"` Endpoint string `json:"endpoint"` Model string `json:"model"` Temperature float64 `json:"temperature"` APIKeyEnv string `json:"api_key_env"` ExtraParams map[string]any `json:"extra_params"` } `json:"effective_model_params"` } run := func(profileID string) (int, string, string, profileReport) { t.Helper() var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"config", "profile", "--config", configPath, "--config-overlay", "development", profileID}, &stdout, &stderr, opts) var report profileReport if code == 0 { if bytes.Count(stdout.Bytes(), []byte("\n")) != 1 { t.Fatalf("profile JSON output = %q, want one object and newline", stdout.String()) } if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { t.Fatalf("decode profile JSON %q: %v", stdout.String(), err) } } return code, stdout.String(), stderr.String(), report } code, output, diagnostic, selected := run("selected") if code != 0 || diagnostic != "" || selected.ProfileID != "selected" || !selected.APIKeyRequired || selected.Target.BackendID != "managed" || selected.Target.Endpoint != "https://provider.example.invalid/v1" || selected.Target.Model != "inherited-model" || selected.Target.Temperature != 0 || selected.Target.APIKeyEnv != "NOT_SET_SYNTHETIC_PROFILE_KEY" || selected.Target.ExtraParams != nil { t.Fatalf("resolved inherited profile = %d %q %q %#v", code, output, diagnostic, selected) } code, output, diagnostic, inherited := run("inherited") if code != 0 || diagnostic != "" || inherited.Target.ExtraParams["Authorization"] != "[REDACTED]" || inherited.Target.ExtraParams["ordinary"] != "visible-value" || strings.Contains(output, "private-authorization") { t.Fatalf("redacted profile = %d %q %q %#v", code, output, diagnostic, inherited) } code, output, diagnostic, endpointOnly := run("endpoint-only") if code != 0 || diagnostic != "" || endpointOnly.ProfileID != "endpoint-only" || endpointOnly.APIKeyRequired || endpointOnly.Target.BackendID != "" || endpointOnly.Target.Endpoint != "https://endpoint.example.invalid/v1" { t.Fatalf("endpoint-only profile = %d %q %q %#v", code, output, diagnostic, endpointOnly) } code, output, diagnostic, literal := run(" literal profile ") if code != 0 || diagnostic != "" || literal.ProfileID != " literal profile " || literal.Target.Model != "literal-model" { t.Fatalf("literal profile ID = %d %q %q %#v", code, output, diagnostic, literal) } if lookupCalls != 0 || factoryCalls != 0 { t.Fatalf("profile inspection called credential lookup %d times and LLM factory %d times", lookupCalls, factoryCalls) } } func TestConfigProfileClassifiesSyntaxAndRuntimeFailures(t *testing.T) { t.Parallel() configPath := writeCommandConfig(t, "session", "other") for _, test := range []struct { name string args []string want int }{ {name: "missing ID", args: []string{"config", "profile", "--config", configPath}, want: 2}, {name: "blank ID", args: []string{"config", "profile", "--config", configPath, " "}, want: 2}, {name: "multiple IDs", args: []string{"config", "profile", "--config", configPath, "one", "two"}, want: 2}, {name: "unknown profile", args: []string{"config", "profile", "--config", configPath, "unknown-profile"}, want: 1}, } { t.Run(test.name, func(t *testing.T) { var stdout, stderr bytes.Buffer if code := RunWithOptions(test.args, &stdout, &stderr, commandContractOptions(t)); code != test.want || stderr.Len() == 0 { t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want) } }) } profilePath := filepath.Join(t.TempDir(), "profiles.yml") writeConfigKitFile(t, profilePath, "profiles:\n writer:\n endpoint: https://example.invalid/v1\n model: writer-model\n") configPath = writeCommandConfigContent(t, "version: 5\npromptkit:\n profile_file: "+profilePath+"\n") var stderr bytes.Buffer if code := RunWithOptions([]string{"config", "profile", "--config", configPath, "writer"}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write PromptKit profile inspection") { t.Fatalf("profile writer failure = %d, %q", code, stderr.String()) } } type failingInspectionWriter struct{} func (failingInspectionWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }