Files
notarius/internal/framework/llm/promptkit_inspection_test.go

144 lines
6.6 KiB
Go

package llm
import (
"encoding/json"
"errors"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
type promptKitInspectionErrorWriter struct{ err error }
func (w promptKitInspectionErrorWriter) Write([]byte) (int, error) { return 0, w.err }
func TestPromptKitProfileInspectionCopiesAndRedactsTarget(t *testing.T) {
const (
profileID = "inspection/profile~case"
keyEnv = "NOT_SET_SYNTHETIC_PROFILE_KEY"
)
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
params := map[string]any{
"Authorization": "Bearer authorization-secret",
"api_key": "api-key-secret",
"ApiKey": "api-key-camel-secret",
"apikey": "apikey-secret",
"credential": "credential-secret",
"credentials": "credentials-secret",
"password": "password-secret",
"secret": "secret-secret",
"token": "token-secret",
"access_token": "access-token-secret",
"normal": "visible-value",
"nested": map[string]any{
"Access_Token": "nested-token-secret",
"items": []any{map[string]any{"PASSWORD": "list-secret", "label": "ordinary"}},
},
}
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
profileID: {Endpoint: &endpoint, Model: &initialModel, APIKeyEnv: promptKitTestString(keyEnv), ExtraParams: params},
})}}},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
profileID: {Model: &finalModel},
})}}},
},
}})
if err != nil {
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
}
profile, err := inspector.InspectProfile(t.Context(), profileID)
if err != nil {
t.Fatalf("InspectProfile() error = %v; an unset synthetic credential name must not be resolved", err)
}
if profile.ProfileID != profileID || profile.BackendID != "" || profile.Model != finalModel || profile.CredentialEnvironment != keyEnv || profile.CredentialRequired {
t.Fatalf("profile inspection = %#v", profile)
}
paramsCopy := profile.EffectiveModelParams.ExtraParams
for _, key := range []string{"Authorization", "api_key", "ApiKey", "apikey", "credential", "credentials", "password", "secret", "token", "access_token"} {
if paramsCopy[key] != "[REDACTED]" {
t.Errorf("effective provider parameter %q = %#v, want redacted", key, paramsCopy[key])
}
}
if paramsCopy["normal"] != "visible-value" {
t.Fatalf("top-level effective provider parameters = %#v", paramsCopy)
}
nested := paramsCopy["nested"].(map[string]any)
if nested["Access_Token"] != "[REDACTED]" {
t.Fatalf("nested effective provider parameters = %#v", nested)
}
items := nested["items"].([]any)
if items[0].(map[string]any)["PASSWORD"] != "[REDACTED]" || items[0].(map[string]any)["label"] != "ordinary" {
t.Fatalf("list provider parameters = %#v", items)
}
encoded, err := json.Marshal(profile.EffectiveModelParams)
if err != nil {
t.Fatal(err)
}
for _, secret := range []string{"authorization-secret", "api-key-secret", "api-key-camel-secret", "apikey-secret", "credential-secret", "credentials-secret", "password-secret", "secret-secret", "token-secret", "access-token-secret", "nested-token-secret", "list-secret"} {
if strings.Contains(string(encoded), secret) {
t.Fatalf("profile reporting projection contains %q", secret)
}
}
nested["ordinary"] = "caller mutation"
paramsCopy["normal"] = "caller mutation"
second, err := inspector.InspectProfile(t.Context(), profileID)
if err != nil {
t.Fatal(err)
}
if second.EffectiveModelParams.ExtraParams["normal"] != "visible-value" || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["ordinary"] != nil {
t.Fatalf("profile inspection retained caller mutation: %#v", second.EffectiveModelParams.ExtraParams)
}
}
func TestPromptKitConfigurationInspectionDistinguishesInvalidAndUnknownPointers(t *testing.T) {
const profileID = "inspection/profile~case"
endpoint, initialModel, finalModel := "https://example.invalid/v1", "model-before", "model-after"
inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower profile", map[string]promptkit.ProfileConfig{
profileID: {Endpoint: &endpoint, Model: &initialModel},
})}}},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile overlay", map[string]promptkit.ProfileConfig{
profileID: {Model: &finalModel},
})}}},
},
}})
if err != nil {
t.Fatalf("NewPromptKitProfileInspector() error = %v", err)
}
report, err := inspector.EffectiveConfiguration()
if err != nil {
t.Fatalf("EffectiveConfiguration() error = %v", err)
}
if !strings.Contains(report, "model-after") || strings.Contains(report, "model-before") {
t.Fatalf("effective report = %q, want only the effective model", report)
}
const escapedPointer = "/profiles/inspection~1profile~0case/model"
explanation, found, err := inspector.ExplainConfiguration(escapedPointer)
if err != nil || !found || !strings.Contains(explanation, "model-before") || !strings.Contains(explanation, "model-after") {
t.Fatalf("ExplainConfiguration(%q) = (%q, %t, %v), want effective and historical values", escapedPointer, explanation, found, err)
}
if _, found, err := inspector.ExplainConfiguration("/profiles/not-configured"); err != nil || found {
t.Fatalf("unknown configuration pointer = (found %t, error %v), want (false, nil)", found, err)
}
if _, _, err := inspector.ExplainConfiguration("/profiles/~2/model"); !errors.Is(err, ErrInvalidPromptKitConfigurationPointer) {
t.Fatalf("invalid configuration pointer error = %v", err)
}
if _, found, err := inspector.ExplainConfiguration("/profiles/inspection~01profile~00case/model"); err != nil || found {
t.Fatalf("configuration pointer with literal escape sequences = (found %t, error %v), want a valid unknown pointer", found, err)
}
writerFailure := errors.New("writer failure")
if err := inspector.WriteEffectiveConfiguration(promptKitInspectionErrorWriter{err: writerFailure}); !errors.Is(err, writerFailure) {
t.Fatalf("WriteEffectiveConfiguration() error = %v, want writer cause", err)
}
if _, err := inspector.WriteConfigurationExplanation(promptKitInspectionErrorWriter{err: writerFailure}, escapedPointer); !errors.Is(err, writerFailure) {
t.Fatalf("WriteConfigurationExplanation() error = %v, want writer cause", err)
}
}