Add scoped PromptKit configuration reports
This commit is contained in:
@@ -276,6 +276,102 @@ func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitScopedInspectionCommandsUseSelectedSources(t *testing.T) {
|
||||
t.Parallel()
|
||||
root := t.TempDir()
|
||||
configPath := filepath.Join(root, "notarius.yml")
|
||||
writeConfigKitFile(t, configPath, "version: 5\npromptkit:\n profile_file: profiles/base.yml\n")
|
||||
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "promptkit:\n profile_file: "+filepath.Join(root, "profiles", "imported.yml")+"\n")
|
||||
writeConfigKitFile(t, filepath.Join(root, "profiles", "base.yml"), `profiles:
|
||||
base:
|
||||
endpoint: https://example.invalid/v1
|
||||
model: base-model
|
||||
extra_params:
|
||||
Authorization: base-auth-secret
|
||||
ordinary: visible-value
|
||||
`)
|
||||
writeConfigKitFile(t, filepath.Join(root, "profiles", "imported.yml"), `configkit:
|
||||
imports:
|
||||
- path: base.yml
|
||||
profiles:
|
||||
"escaped/id":
|
||||
model: imported-model
|
||||
dnd-extraction:
|
||||
model: imported-override
|
||||
`)
|
||||
assets, err := productionPromptAssets()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
factoryCalls := 0
|
||||
opts := commandContractOptions(t)
|
||||
opts.promptKitAssets = assets
|
||||
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||
factoryCalls++
|
||||
return nil, nil, errors.New("inspection must not construct an LLM client")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"config", "effective", "--config", configPath}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "pipelines:") || strings.Contains(stdout.String(), "profiles:") || stderr.Len() != 0 {
|
||||
t.Fatalf("default effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
defaultReport := stdout.String()
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "effective", "--scope", "notarius", "--config", configPath}, &stdout, &stderr, opts); code != 0 || stdout.String() != defaultReport || stderr.Len() != 0 {
|
||||
t.Fatalf("explicit Notarius effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || strings.Contains(stdout.String(), "base-auth-secret") || !strings.Contains(stdout.String(), "visible-value") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit effective report = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/escaped~1id/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-model") || !strings.Contains(stdout.String(), "imported.yml") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit imported explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/base/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "base-model") || !strings.Contains(stdout.String(), "base.yml") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit imported base explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
if code := RunWithOptions([]string{"config", "explain", "--scope", "promptkit", "--config", configPath, "--config-overlay", "development", "/profiles/dnd-extraction/model"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "imported-override") || !strings.Contains(stdout.String(), "dnd-extraction.yaml") || stderr.Len() != 0 {
|
||||
t.Fatalf("PromptKit historical fallback explanation = %d, %q, %q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
if factoryCalls != 0 {
|
||||
t.Fatalf("inspection invoked LLM client factory %d times", factoryCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitScopedInspectionClassifiesSelectorsAndWriterErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
path := writeCommandConfig(t, "session", "other")
|
||||
opts := commandContractOptions(t)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
want int
|
||||
}{
|
||||
{name: "blank scope", args: []string{"config", "effective", "--scope="}, want: 2},
|
||||
{name: "unknown scope", args: []string{"config", "effective", "--scope", "unknown"}, want: 2},
|
||||
{name: "malformed pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/profiles/~2"}, want: 2},
|
||||
{name: "unknown pointer", args: []string{"config", "explain", "--scope", "promptkit", "--config", path, "/not-configured"}, want: 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := RunWithOptions(test.args, &stdout, &stderr, opts); code != test.want || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q, want exit %d and diagnostic", code, stdout.String(), stderr.String(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
if code := RunWithOptions([]string{"config", "effective", "--scope", "promptkit", "--config", path}, failingInspectionWriter{}, &stderr, opts); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
||||
t.Fatalf("PromptKit writer failure = %d, %q", code, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
type failingInspectionWriter struct{}
|
||||
|
||||
func (failingInspectionWriter) Write([]byte) (int, error) {
|
||||
|
||||
Reference in New Issue
Block a user