Capture immutable PromptKit configuration identity
This commit is contained in:
@@ -401,7 +401,7 @@ func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.
|
||||
})
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
|
||||
writeProfile := func(model string) {
|
||||
@@ -411,7 +411,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
fingerprintFor := func() CheckpointFingerprint {
|
||||
clientFor := func() *PromptKitClient {
|
||||
t.Helper()
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
@@ -420,34 +420,42 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
fingerprintFor := func(client *PromptKitClient) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
values, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values)
|
||||
if len(values) != 1 || values[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", values)
|
||||
}
|
||||
return values[0]
|
||||
}
|
||||
|
||||
writeProfile("model-one")
|
||||
first := fingerprintFor()
|
||||
repeated := fingerprintFor()
|
||||
client := clientFor()
|
||||
first := fingerprintFor(client)
|
||||
repeated := fingerprintFor(clientFor())
|
||||
if first != repeated {
|
||||
t.Fatalf("profile-source fingerprint = %#v then %#v for unchanged source", first, repeated)
|
||||
t.Fatalf("configuration fingerprint = %#v then %#v for unchanged source", first, repeated)
|
||||
}
|
||||
writeProfile("model-two")
|
||||
second := fingerprintFor()
|
||||
if captured := fingerprintFor(client); captured != first {
|
||||
t.Fatalf("existing engine fingerprint after source edit = %#v, want %#v", captured, first)
|
||||
}
|
||||
second := fingerprintFor(clientFor())
|
||||
if first == second {
|
||||
t.Fatalf("profile-source fingerprint = %#v for both profile models", first)
|
||||
t.Fatalf("configuration fingerprint = %#v for both profile models", first)
|
||||
}
|
||||
if strings.TrimSpace(first.Value) == "" {
|
||||
t.Fatal("profile-source fingerprint is empty")
|
||||
t.Fatal("configuration fingerprint is empty")
|
||||
}
|
||||
if strings.Contains(first.Value, profilePath) ||
|
||||
strings.Contains(first.Value, "model-one") ||
|
||||
strings.Contains(first.Value, credentialEnvironment) {
|
||||
t.Fatalf("profile-source fingerprint exposes source details: %#v", first)
|
||||
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
|
||||
}
|
||||
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)})
|
||||
@@ -475,92 +483,12 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(secondBuiltin) != 1 || fresh[0] != secondBuiltin[0] {
|
||||
t.Fatalf("built-in profile fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
|
||||
t.Fatalf("built-in configuration fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
|
||||
}
|
||||
if strings.TrimSpace(fresh[0].Value) == "" {
|
||||
t.Fatal("built-in profile fingerprint is empty")
|
||||
t.Fatal("built-in configuration fingerprint is empty")
|
||||
}
|
||||
|
||||
t.Run("directory layout", func(t *testing.T) {
|
||||
profileDir := t.TempDir()
|
||||
firstPath := filepath.Join(profileDir, "first-profile.yaml")
|
||||
secondPath := filepath.Join(profileDir, "second-profile.yaml")
|
||||
content := []byte("profiles:\n directory-profile:\n endpoint: http://promptkit.test/v1\n model: directory-model\n")
|
||||
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(firstPath, secondPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatalf("profile-source fingerprint = %#v after source filename changed", first)
|
||||
}
|
||||
if strings.Contains(first.Value, firstPath) || strings.Contains(second.Value, secondPath) {
|
||||
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inherited parent", func(t *testing.T) {
|
||||
profileDir := t.TempDir()
|
||||
parentPath := filepath.Join(profileDir, "parent.yaml")
|
||||
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
||||
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-one\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(leafPath, []byte("profiles:\n leaf:\n extends: parent\n model: leaf-model\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-two\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
|
||||
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
||||
for _, source := range []struct {
|
||||
name string
|
||||
profileDir string
|
||||
profileFile string
|
||||
}{
|
||||
{
|
||||
name: "file",
|
||||
profileFile: filepath.Join(t.TempDir(), "missing-profile.yaml"),
|
||||
},
|
||||
{
|
||||
name: "directory",
|
||||
profileDir: filepath.Join(t.TempDir(), "missing-profiles"),
|
||||
},
|
||||
} {
|
||||
t.Run(source.name, func(t *testing.T) {
|
||||
_, err := promptKitProfileFingerprint(source.profileDir, source.profileFile, "")
|
||||
if err == nil {
|
||||
t.Fatal("promptKitProfileFingerprint() error = nil, want source read failure")
|
||||
}
|
||||
if (source.profileDir != "" && strings.Contains(err.Error(), source.profileDir)) ||
|
||||
(source.profileFile != "" && strings.Contains(err.Error(), source.profileFile)) {
|
||||
t.Fatalf("fingerprint error exposes profile source path: %q", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
|
||||
@@ -749,8 +677,8 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want captured configuration identity", fingerprints)
|
||||
}
|
||||
return fingerprints[0]
|
||||
}
|
||||
@@ -765,15 +693,82 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
|
||||
fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
assets := NewAssetRegistry()
|
||||
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
|
||||
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||
"prompt.yaml": {Data: []byte(promptDocument)},
|
||||
}, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := assets.RegisterSchemaFS(fstest.MapFS{
|
||||
"result.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoint, model := "https://provider.example.test/v1", "fingerprint-model"
|
||||
required := keyRequired
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("fingerprint profile", map[string]promptkit.ProfileConfig{
|
||||
"fingerprint-profile": {
|
||||
Endpoint: &endpoint,
|
||||
Model: &model,
|
||||
APIKeyRequired: &required,
|
||||
ExtraParams: map[string]any{"access_token": parameterValue},
|
||||
},
|
||||
}),
|
||||
}}}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||
}
|
||||
fingerprints, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatalf("LLMCheckpointFingerprints() error = %v", err)
|
||||
}
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v", fingerprints)
|
||||
}
|
||||
return fingerprints[0]
|
||||
}
|
||||
|
||||
baseline := fingerprintFor("original message", "provider-secret-one", false)
|
||||
if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed {
|
||||
t.Fatal("captured prompt message change did not alter configuration fingerprint")
|
||||
}
|
||||
if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed {
|
||||
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
|
||||
}
|
||||
if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed {
|
||||
t.Fatal("credential-policy change did not alter configuration fingerprint")
|
||||
}
|
||||
if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") {
|
||||
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
|
||||
const (
|
||||
firstEndpoint = "http://localhost:8000/v1"
|
||||
secondEndpoint = "https://inference.example.test/v1"
|
||||
)
|
||||
fingerprintsFor := func(endpoint string, concurrencyLimit int) []CheckpointFingerprint {
|
||||
fingerprintsFor := func(endpoint string, concurrencyLimit, queueCapacity int) []CheckpointFingerprint {
|
||||
t.Helper()
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||
Source: promptkit.BackendSource("operator scheduling", map[string]promptkit.BackendConfig{
|
||||
promptkit.BackendLocal: {
|
||||
Endpoint: promptKitTestString("http://operator-local.test/v1"),
|
||||
ConcurrencyLimit: promptKitTestInt(1),
|
||||
QueueCapacity: &queueCapacity,
|
||||
},
|
||||
}),
|
||||
}}}}},
|
||||
LocalBackend: &PromptKitLocalBackendConfig{
|
||||
Endpoint: endpoint,
|
||||
ConcurrencyLimit: concurrencyLimit,
|
||||
@@ -789,20 +784,22 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
||||
return values
|
||||
}
|
||||
|
||||
baseline := fingerprintsFor(firstEndpoint, 0)
|
||||
if len(baseline) != 2 ||
|
||||
baseline[0].Name != promptKitProfileFingerprintName ||
|
||||
baseline[1].Name != promptKitLocalBackendFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source then local backend target", baseline)
|
||||
baseline := fingerprintsFor(firstEndpoint, 2, 1)
|
||||
if len(baseline) != 1 || baseline[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", baseline)
|
||||
}
|
||||
endpointChanged := fingerprintsFor(secondEndpoint, 0)
|
||||
if baseline[0] != endpointChanged[0] || baseline[1] == endpointChanged[1] {
|
||||
t.Fatalf("endpoint fingerprints = %#v and %#v, want only local target to change", baseline, endpointChanged)
|
||||
endpointChanged := fingerprintsFor(secondEndpoint, 2, 1)
|
||||
if baseline[0] == endpointChanged[0] {
|
||||
t.Fatalf("configuration fingerprints = %#v and %#v, want endpoint change to affect identity", baseline, endpointChanged)
|
||||
}
|
||||
concurrencyChanged := fingerprintsFor(firstEndpoint, 4)
|
||||
concurrencyChanged := fingerprintsFor(firstEndpoint, 4, 1)
|
||||
if !reflect.DeepEqual(baseline, concurrencyChanged) {
|
||||
t.Fatalf("concurrency fingerprints = %#v, want %#v", concurrencyChanged, baseline)
|
||||
}
|
||||
queueCapacityChanged := fingerprintsFor(firstEndpoint, 2, 2)
|
||||
if !reflect.DeepEqual(baseline, queueCapacityChanged) {
|
||||
t.Fatalf("queue-capacity fingerprints = %#v, want %#v", queueCapacityChanged, baseline)
|
||||
}
|
||||
for _, values := range [][]CheckpointFingerprint{baseline, endpointChanged} {
|
||||
for _, fingerprint := range values {
|
||||
if strings.Contains(fingerprint.Value, firstEndpoint) ||
|
||||
@@ -814,7 +811,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
||||
|
||||
localBackend := &PromptKitLocalBackendConfig{
|
||||
Endpoint: " " + firstEndpoint + " ",
|
||||
ConcurrencyLimit: 0,
|
||||
ConcurrencyLimit: 2,
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
@@ -831,8 +828,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
||||
if !reflect.DeepEqual(copy, baseline) {
|
||||
t.Fatalf("fingerprints after input mutation = %#v, want retained target %#v", copy, baseline)
|
||||
}
|
||||
copy[0].Value = "mutated-profile"
|
||||
copy[1].Value = "mutated-target"
|
||||
copy[0].Value = "mutated-configuration"
|
||||
fresh, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user