Preserve inspection redaction for resource lists
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -19,23 +20,26 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
||||
"backends": []any{map[string]any{
|
||||
"id": "local", "endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
|
||||
"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
|
||||
}},
|
||||
"collections": map[string]any{"literal": map[string]any{
|
||||
"backends": map[string]any{"local": map[string]any{
|
||||
"endpoint": "https://example.invalid/v1",
|
||||
"extra_params": map[string]any{"token": "namespaced-private-sentinel"},
|
||||
}},
|
||||
"profiles": map[string]any{
|
||||
"base.v1": map[string]any{"backend": "local", "model": "default-model"},
|
||||
"id": map[string]any{"extends": "base.v1"},
|
||||
"backends": []any{
|
||||
map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "namespaced-private-sentinel"}},
|
||||
map[string]any{"id": "child", "extends": map[string]any{"id": "local"}},
|
||||
},
|
||||
"profiles": []any{
|
||||
map[string]any{"id": "base.v1", "backend": "local", "model": "default-model"},
|
||||
map[string]any{"id": "id", "extends": map[string]any{"id": "base.v1"}},
|
||||
},
|
||||
}},
|
||||
})}}},
|
||||
{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
|
||||
"collections": map[string]any{"literal": map[string]any{"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}}}},
|
||||
"collections": map[string]any{"literal": map[string]any{
|
||||
"backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}},
|
||||
"profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}},
|
||||
}},
|
||||
})}}},
|
||||
},
|
||||
}})
|
||||
@@ -46,29 +50,38 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
||||
if err != nil {
|
||||
t.Fatalf("inspect configuration: %v", err)
|
||||
}
|
||||
modelPath := configkit.Keys("collections", "literal", "profiles", "base.v1", "model")
|
||||
modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model")
|
||||
model, found := view.Explain(modelPath)
|
||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
|
||||
if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 {
|
||||
t.Fatalf("layered model explanation = (%+v, %t)", model, found)
|
||||
}
|
||||
inheritedPath := configkit.Keys("collections", "literal", "profiles", "id", "model")
|
||||
inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model")
|
||||
inherited, found := view.Explain(inheritedPath)
|
||||
if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
|
||||
t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
|
||||
}
|
||||
if inheritedPath.String() != "/collections/literal/profiles/id/model" {
|
||||
if inheritedPath.String() != "/collections/literal/profiles/1/model" {
|
||||
t.Fatalf("literal-key path = %q", inheritedPath)
|
||||
}
|
||||
secretPath := configkit.Keys("backends", "local", "extra_params", "token")
|
||||
secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token")
|
||||
secret, found := view.Explain(secretPath)
|
||||
if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
|
||||
t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
|
||||
}
|
||||
namespacedSecretPath := configkit.Keys("collections", "literal", "backends", "local", "extra_params", "token")
|
||||
namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token")
|
||||
namespacedSecret, found := view.Explain(namespacedSecretPath)
|
||||
if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") {
|
||||
t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found)
|
||||
}
|
||||
inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token")
|
||||
inheritedSecret, found := view.Explain(inheritedSecretPath)
|
||||
if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") {
|
||||
t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found)
|
||||
}
|
||||
metadata := fmt.Sprintf("%+v", view.Metadata().History(inheritedSecretPath))
|
||||
if strings.Contains(metadata, "replacement-private-sentinel") || strings.Contains(metadata, "namespaced-private-sentinel") {
|
||||
t.Fatal("metadata disclosed a sensitive provider value")
|
||||
}
|
||||
var effective bytes.Buffer
|
||||
if err := view.WriteEffective(&effective); err != nil {
|
||||
t.Fatalf("write effective configuration: %v", err)
|
||||
@@ -104,9 +117,26 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) {
|
||||
_, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("invalid backend", map[string]any{
|
||||
"collections": map[string]any{"private": map[string]any{"backends": []any{
|
||||
map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}},
|
||||
}}},
|
||||
}),
|
||||
}}})
|
||||
var loadErr *configkit.LoadError
|
||||
if !errors.As(err, &loadErr) {
|
||||
t.Fatalf("invalid backend error = %v, want load error", err)
|
||||
}
|
||||
if text := fmt.Sprintf("%v %+v", err, loadErr.Diagnostics()); strings.Contains(text, "failure-private-sentinel") {
|
||||
t.Fatalf("failure diagnostics disclosed a sensitive value: %s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
|
||||
Source: configkit.Mapping("main", map[string]any{"version": "1"}),
|
||||
Source: configkit.Mapping("main", map[string]any{"version": "2"}),
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with accepted declaration: %v", err)
|
||||
@@ -117,7 +147,7 @@ func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T
|
||||
}
|
||||
path := configkit.Keys("version")
|
||||
field, found := inspection.Explain(path)
|
||||
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "1") {
|
||||
if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") {
|
||||
t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found)
|
||||
}
|
||||
var effective bytes.Buffer
|
||||
@@ -150,9 +180,8 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
||||
t.Fatalf("zero explanation writer = %v", err)
|
||||
}
|
||||
|
||||
endpoint, model := "https://configured.example/v1", "configured-model"
|
||||
profile := configkit.Input{Source: promptkit.ProfileSource("inspection test profile", map[string]promptkit.ProfileConfig{
|
||||
"inspection-profile": {Endpoint: &endpoint, Model: &model},
|
||||
profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}},
|
||||
})}
|
||||
valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}},
|
||||
@@ -169,14 +198,15 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
|
||||
if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("effective writer error = %v", err)
|
||||
}
|
||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles", "inspection-profile", "model")); !errors.Is(err, writeErr) {
|
||||
if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) {
|
||||
t.Fatalf("explanation writer error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
|
||||
const promptDocument = "prompts:\n - id: captured\n default_profile: offline\n messages:\n - role: user\n content_file: body.tmpl\n output:\n format: text\n validation_mode: none\n"
|
||||
source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||
}}
|
||||
inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
|
||||
@@ -184,7 +214,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
t.Fatalf("select inputs: %v", err)
|
||||
}
|
||||
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||
"config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)},
|
||||
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||
}}
|
||||
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||
@@ -195,16 +225,16 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
client := countingLLMClient{calls: &modelCalls}
|
||||
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &model},
|
||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
||||
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
||||
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{
|
||||
"profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}},
|
||||
})}}}},
|
||||
}}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
@@ -226,7 +256,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
if err != nil || len(differences) != 1 {
|
||||
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||
}
|
||||
if differences[0].Path.String() != "/profiles/offline/model" ||
|
||||
if differences[0].Path.String() != "/profiles" ||
|
||||
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||
t.Fatalf("captured difference = %+v", differences[0])
|
||||
@@ -238,7 +268,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
||||
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||
}
|
||||
}
|
||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "messages").Index(0).Key("content_file")); !found {
|
||||
if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found {
|
||||
t.Fatal("captured prompt content path was not inspectable")
|
||||
}
|
||||
var report bytes.Buffer
|
||||
|
||||
Reference in New Issue
Block a user