package cli import ( "bytes" "encoding/json" "errors" "fmt" "os" "path/filepath" "strings" "testing" configkit "gitea.maximumdirect.net/eric/ConfigKit" renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format" ) func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) { configPath := writeAppConfigFile(t, "{}\n") code, stdout, stderr := runCLICommand(t, inspectCommand, []string{ "configuration", "validate", "--config", configPath, }) if code != ExitOK || stdout != "" || stderr != "" { t.Fatalf("source-free validation failed: code=%d stdout=%q stderr=%q", code, stdout, stderr) } t.Setenv("SCRIPTORIUM_FRAMEWORK_KEY", "") root := t.TempDir() frameworkFile := filepath.Join(root, "framework.yml") writeFrameworkSourceFile(t, root, "framework.yml", `backends: local: endpoint: http://127.0.0.1:1/v1 api_key_env: SCRIPTORIUM_FRAMEWORK_KEY profiles: protected: backend: local model: offline-model api_key_required: true `) configPath = writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) for _, mode := range []string{"validate", "show", "sources"} { code, stdout, stderr = runCLICommand(t, inspectCommand, []string{ "configuration", mode, "--config", configPath, }) if code != ExitOK || stderr != "" { t.Fatalf("%s required a prompt or credential: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr) } if mode == "validate" && stdout != "" { t.Fatalf("validation wrote success output: %q", stdout) } if mode == "show" && (!strings.Contains(stdout, "profiles:") || !strings.Contains(stdout, "offline-model")) { t.Fatalf("effective framework YAML is missing: %q", stdout) } } } func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T) { root := t.TempDir() importDir := filepath.Join(root, "catalog") if err := os.MkdirAll(importDir, 0o755); err != nil { t.Fatal(err) } mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit: imports: - path: catalog/fragment.yml backends: local: endpoint: http://127.0.0.1:1/v1 profiles: selected: backend: local model: selected-model `) fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles: imported: model: imported-model `) directory := t.TempDir() directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles: directory: model: directory-model `) configuredMissing := filepath.Join(root, "configured-but-replaced.yml") configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", configuredMissing)) args := []string{ "configuration", "sources", "--config", configPath, "--promptkit-source", mainPath, "--promptkit-source", directory, } code, stdout, stderr := runCLICommand(t, inspectCommand, args) if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") { t.Fatalf("framework sources command failed: code=%d stdout=%q stderr=%q", code, stdout, stderr) } var sources []frameworkConfigurationSource if err := json.Unmarshal([]byte(stdout), &sources); err != nil { t.Fatalf("decode framework source DTOs: %v\n%s", err, stdout) } var rawSources []map[string]any if err := json.Unmarshal([]byte(stdout), &rawSources); err != nil { t.Fatalf("decode framework source fields: %v", err) } mainIndex := findFrameworkSource(t, sources, mainPath) fragmentIndex := findFrameworkSource(t, sources, fragmentPath) directoryIndex := findFrameworkSource(t, sources, directoryPath) main := sources[mainIndex] fragment := sources[fragmentIndex] directorySource := sources[directoryIndex] if !(mainIndex < fragmentIndex && fragmentIndex < directoryIndex) { t.Fatalf("captured source order changed: main=%d fragment=%d directory=%d", mainIndex, fragmentIndex, directoryIndex) } if main.ID == "" || main.Label == "" || main.Layer != configkit.LayerUnderlay || main.Kind != configkit.SourceKindFile || main.UnderlayIndex < 1 { t.Fatalf("explicit file source fields are incomplete: %+v", main) } for _, key := range []string{"id", "label", "layer", "kind", "underlay_index", "path"} { if _, exists := rawSources[mainIndex][key]; !exists { t.Fatalf("source DTO omitted %q: %#v", key, rawSources[mainIndex]) } } for _, key := range []string{"ID", "Label", "Layer", "Kind", "UnderlayIndex", "filesystem"} { if _, exists := rawSources[mainIndex][key]; exists { t.Fatalf("source DTO exposed implementation field %q: %#v", key, rawSources[mainIndex]) } } if fragment.Layer != configkit.LayerUnderlay || fragment.Kind != configkit.SourceKindFile || fragment.ImportedFrom == nil || fragment.ImportedFrom.SourceID != main.ID || fragment.ImportedFrom.Line < 1 || fragment.ImportedFrom.Column < 1 { t.Fatalf("import source fields are incomplete: %+v", fragment) } if importedFrom, ok := rawSources[fragmentIndex]["imported_from"].(map[string]any); !ok || importedFrom["source_id"] != main.ID || importedFrom["line"] == nil || importedFrom["column"] == nil { t.Fatalf("import location DTO fields are incomplete: %#v", rawSources[fragmentIndex]) } if directorySource.Layer != configkit.LayerUnderlay || directorySource.Kind != configkit.SourceKindFile || directorySource.UnderlayIndex <= main.UnderlayIndex { t.Fatalf("directory-selected source fields/order are incomplete: %+v (main %+v)", directorySource, main) } } func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) { root := t.TempDir() frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts: unused: "1": messages: - role: user content: "{{" output: format: text validation_mode: none `) configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) code, stdout, stderr := runCLICommand(t, configCommand, []string{"validate", "--config", configPath}) if code != ExitOK || stdout != "" || stderr != "" { t.Fatalf("application validation loaded framework definitions: code=%d stdout=%q stderr=%q", code, stdout, stderr) } for _, args := range [][]string{ {"configuration", "validate", "--config", configPath}, {"configuration", "show", "--config", configPath}, {"configuration", "sources", "--config", configPath}, {"configuration", "explain", "--config", configPath, "--path", ""}, } { code, stdout, stderr = runCLICommand(t, inspectCommand, args) if code != ExitRuntimeError || stdout != "" || stderr == "" { t.Fatalf("invalid final framework definition was reported as success: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) } } } func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors(t *testing.T) { configPath := writeAppConfigFile(t, "{}\n") for _, args := range [][]string{ {"configuration"}, {"configuration", "explain"}, {"configuration", "show", "unexpected"}, {"configuration", "sources", "--format", "json"}, } { code, stdout, stderr := runCLICommand(t, inspectCommand, args) if code != ExitRuntimeError || stdout != "" || stderr == "" { t.Fatalf("invalid inspection arguments were accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) } } writerError := errors.New("synthetic framework output failure") var stderr bytes.Buffer code := inspectCommand([]string{"configuration", "show", "--config", configPath}, frameworkConfigurationFailingWriter{err: writerError}, &stderr) if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) { t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String()) } stderr.Reset() code = inspectCommand([]string{"configuration", "explain", "--config", configPath, "--path", ""}, frameworkConfigurationFailingWriter{err: writerError}, &stderr) if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) { t.Fatalf("explanation writer failure was not reported: code=%d stderr=%q", code, stderr.String()) } } func TestFrameworkConfigurationExplainUsesLiteralJSONPointersAndNativeHistory(t *testing.T) { root := t.TempDir() lower := writeFrameworkSourceFile(t, root, "lower.yml", `backends: local: endpoint: http://127.0.0.1:1/v1 api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE extra_params: token: backend-token-displaced-canary password: backend-password-canary visible_option: backend-visible-value provider.option: dotted-visible-value profiles: base: backend: local model: inherited-model api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE extra_params: token: profile-token-displaced-canary password: profile-password-displaced-canary visible_option: profile-visible-value provider.option: dotted-visible-value "team/summary": extends: base model: lower-model prompts: explainable: "1": default_profile: "team/summary" messages: - role: user content: hello output: format: text validation_mode: none `) upperRoot := t.TempDir() upper := writeFrameworkSourceFile(t, upperRoot, "upper.yml", `backends: local: endpoint: http://127.0.0.1:1/v1 api_key_env: SCRIPTORIUM_EXPLAIN_REFERENCE extra_params: token: backend-token-effective-canary password: backend-password-effective-canary visible_option: backend-visible-value provider.option: dotted-visible-value profiles: "team/summary": model: effective-model extra_params: token: profile-token-effective-canary password: profile-password-effective-canary visible_option: profile-visible-value provider.option: dotted-visible-value `) configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n - file: %q\n", lower, upper)) const secret = "environment-value-must-never-be-emitted" t.Setenv("SCRIPTORIUM_EXPLAIN_REFERENCE", secret) for _, tc := range []struct { path string want []string unwant []string }{ { path: "/backends/local/extra_params/token", unwant: []string{"backend-token-displaced-canary", "backend-token-effective-canary"}, }, { path: "/backends/local/extra_params/password", unwant: []string{"backend-password-displaced-canary", "backend-password-effective-canary"}, }, { path: "/profiles/team~1summary/model", want: []string{"effective-model", "inherited-model", "lower-model"}, unwant: []string{}, }, { path: "/profiles/team~1summary/extra_params/token", unwant: []string{"profile-token-displaced-canary", "profile-token-effective-canary"}, }, { path: "/profiles/team~1summary/extra_params/password", unwant: []string{"profile-password-displaced-canary", "profile-password-effective-canary"}, }, { path: "/profiles/team~1summary/api_key_env", want: []string{"SCRIPTORIUM_EXPLAIN_REFERENCE"}, unwant: []string{secret}, }, { path: "/profiles/team~1summary/extra_params/provider.option", want: []string{"dotted-visible-value"}, unwant: []string{"provider/option"}, }, { path: "", want: []string{"profiles:", "effective-model"}, unwant: []string{"profile-token-displaced-canary", "backend-password-effective-canary"}, }, } { args := []string{"configuration", "explain", "--config", configPath, "--path", tc.path} code, stdout, stderr := runCLICommand(t, inspectCommand, args) if code != ExitOK || stderr != "" { t.Fatalf("explain %q failed: code=%d stdout=%q stderr=%q", tc.path, code, stdout, stderr) } if stdout == "" { t.Fatalf("explain %q produced no output", tc.path) } for _, value := range tc.want { if !strings.Contains(stdout, value) { t.Errorf("explain %q omitted %q: %s", tc.path, value, stdout) } } for _, value := range tc.unwant { if strings.Contains(stdout, value) { t.Errorf("explain %q disclosed %q: %s", tc.path, value, stdout) } } if strings.Contains(stdout, secret) { t.Errorf("explain %q disclosed the resolved environment value: %s", tc.path, stdout) } } code, stdout, stderr := runCLICommand(t, inspectCommand, []string{"configuration", "show", "--config", configPath}) if code != ExitOK || stderr != "" { t.Fatalf("framework show failed: code=%d stderr=%q", code, stderr) } for _, value := range []string{"SCRIPTORIUM_EXPLAIN_REFERENCE", "backend-visible-value", "profile-visible-value", "dotted-visible-value"} { if !strings.Contains(stdout, value) { t.Errorf("effective report omitted visible configuration %q: %s", value, stdout) } } for _, value := range []string{ "backend-token-displaced-canary", "backend-token-effective-canary", "backend-password-displaced-canary", "backend-password-effective-canary", "profile-token-displaced-canary", "profile-token-effective-canary", "profile-password-displaced-canary", "profile-password-effective-canary", secret, } { if strings.Contains(stdout, value) { t.Errorf("effective report disclosed classified or resolved value %q: %s", value, stdout) } } profileCode, profileOutput, profileError := runCLICommand(t, inspectCommand, []string{ "profile", "--config", configPath, "--profile", "team/summary", "--format", "json", }) if profileCode != ExitOK || profileError != "" { t.Fatalf("existing profile inspection failed: code=%d stdout=%q stderr=%q", profileCode, profileOutput, profileError) } var profile renderformat.ProfileInspection if err := json.Unmarshal([]byte(profileOutput), &profile); err != nil { t.Fatalf("decode profile inspection: %v\n%s", err, profileOutput) } if profile.EffectiveModelParams.Model != "effective-model" || profile.EffectiveModelParams.ExtraParams["visible_option"] != "profile-visible-value" { t.Fatalf("existing profile inspection changed configured provider data: %+v", profile.EffectiveModelParams) } if strings.Contains(profileOutput, secret) { t.Fatalf("profile inspection emitted resolved environment data: %s", profileOutput) } code, rendered, stderr := runCLICommand(t, renderCommand, []string{"--config", configPath, "--prompt", "explainable"}) if code != ExitOK || stderr != "" { t.Fatalf("prepared output failed: code=%d stdout=%q stderr=%q", code, rendered, stderr) } if !strings.Contains(rendered, "profile-visible-value") || strings.Contains(rendered, secret) { t.Fatalf("prepared output did not preserve ordinary provider data or disclosed environment data: %s", rendered) } } func TestFrameworkConfigurationExplainRejectsMissingMalformedAndUnknownPaths(t *testing.T) { configPath := writeAppConfigFile(t, "{}\n") for _, args := range [][]string{ {"configuration", "explain"}, {"configuration", "explain", "--path", "not-a-pointer"}, {"configuration", "explain", "--path", "/profiles/team~2summary/model"}, {"configuration", "explain", "--config", configPath, "--path", "/profiles/missing/model"}, } { code, stdout, stderr := runCLICommand(t, inspectCommand, args) if code != ExitRuntimeError || stdout != "" || stderr == "" { t.Fatalf("invalid explanation path was accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr) } } } func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int { t.Helper() for index, source := range sources { if filepath.Clean(source.Path) == filepath.Clean(path) { return index } } t.Fatalf("selected source path %q was not present: %+v", path, sources) return -1 } type frameworkConfigurationFailingWriter struct { err error } func (writer frameworkConfigurationFailingWriter) Write([]byte) (int, error) { return 0, writer.err }