Capture immutable PromptKit configuration identity

This commit is contained in:
2026-10-04 12:59:29 +00:00
parent ef504eabb2
commit deaa3560bc
6 changed files with 259 additions and 248 deletions

View File

@@ -401,7 +401,7 @@ func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.
})
}
func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *testing.T) {
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
writeProfile := func(model string) {
@@ -411,7 +411,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
t.Fatal(err)
}
}
fingerprintFor := func() CheckpointFingerprint {
clientFor := func() *PromptKitClient {
t.Helper()
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
@@ -420,34 +420,42 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
if err != nil {
t.Fatal(err)
}
return client
}
fingerprintFor := func(client *PromptKitClient) CheckpointFingerprint {
t.Helper()
values, err := client.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)
}
if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values)
if len(values) != 1 || values[0].Name != promptKitConfigurationFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", values)
}
return values[0]
}
writeProfile("model-one")
first := fingerprintFor()
repeated := fingerprintFor()
client := clientFor()
first := fingerprintFor(client)
repeated := fingerprintFor(clientFor())
if first != repeated {
t.Fatalf("profile-source fingerprint = %#v then %#v for unchanged source", first, repeated)
t.Fatalf("configuration fingerprint = %#v then %#v for unchanged source", first, repeated)
}
writeProfile("model-two")
second := fingerprintFor()
if captured := fingerprintFor(client); captured != first {
t.Fatalf("existing engine fingerprint after source edit = %#v, want %#v", captured, first)
}
second := fingerprintFor(clientFor())
if first == second {
t.Fatalf("profile-source fingerprint = %#v for both profile models", first)
t.Fatalf("configuration fingerprint = %#v for both profile models", first)
}
if strings.TrimSpace(first.Value) == "" {
t.Fatal("profile-source fingerprint is empty")
t.Fatal("configuration fingerprint is empty")
}
if strings.Contains(first.Value, profilePath) ||
strings.Contains(first.Value, "model-one") ||
strings.Contains(first.Value, credentialEnvironment) {
t.Fatalf("profile-source fingerprint exposes source details: %#v", first)
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
}
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)})
@@ -475,92 +483,12 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
t.Fatal(err)
}
if len(secondBuiltin) != 1 || fresh[0] != secondBuiltin[0] {
t.Fatalf("built-in profile fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
t.Fatalf("built-in configuration fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
}
if strings.TrimSpace(fresh[0].Value) == "" {
t.Fatal("built-in profile fingerprint is empty")
t.Fatal("built-in configuration fingerprint is empty")
}
t.Run("directory layout", func(t *testing.T) {
profileDir := t.TempDir()
firstPath := filepath.Join(profileDir, "first-profile.yaml")
secondPath := filepath.Join(profileDir, "second-profile.yaml")
content := []byte("profiles:\n directory-profile:\n endpoint: http://promptkit.test/v1\n model: directory-model\n")
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
t.Fatal(err)
}
first, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if err := os.Rename(firstPath, secondPath); err != nil {
t.Fatal(err)
}
second, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if first == second {
t.Fatalf("profile-source fingerprint = %#v after source filename changed", first)
}
if strings.Contains(first.Value, firstPath) || strings.Contains(second.Value, secondPath) {
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
}
})
t.Run("inherited parent", func(t *testing.T) {
profileDir := t.TempDir()
parentPath := filepath.Join(profileDir, "parent.yaml")
leafPath := filepath.Join(profileDir, "leaf.yaml")
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-one\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(leafPath, []byte("profiles:\n leaf:\n extends: parent\n model: leaf-model\n"), 0o600); err != nil {
t.Fatal(err)
}
first, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-two\n"), 0o600); err != nil {
t.Fatal(err)
}
second, err := promptKitProfileFingerprint(profileDir, "", "")
if err != nil {
t.Fatal(err)
}
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
}
})
}
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
for _, source := range []struct {
name string
profileDir string
profileFile string
}{
{
name: "file",
profileFile: filepath.Join(t.TempDir(), "missing-profile.yaml"),
},
{
name: "directory",
profileDir: filepath.Join(t.TempDir(), "missing-profiles"),
},
} {
t.Run(source.name, func(t *testing.T) {
_, err := promptKitProfileFingerprint(source.profileDir, source.profileFile, "")
if err == nil {
t.Fatal("promptKitProfileFingerprint() error = nil, want source read failure")
}
if (source.profileDir != "" && strings.Contains(err.Error(), source.profileDir)) ||
(source.profileFile != "" && strings.Contains(err.Error(), source.profileFile)) {
t.Fatalf("fingerprint error exposes profile source path: %q", err)
}
})
}
}
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
@@ -749,8 +677,8 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
if err != nil {
t.Fatal(err)
}
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want captured configuration identity", fingerprints)
}
return fingerprints[0]
}
@@ -765,15 +693,82 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
}
}
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint {
t.Helper()
assets := NewAssetRegistry()
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
if err := assets.RegisterPromptFS(fstest.MapFS{
"prompt.yaml": {Data: []byte(promptDocument)},
}, "."); err != nil {
t.Fatal(err)
}
if err := assets.RegisterSchemaFS(fstest.MapFS{
"result.json": {Data: []byte(`{"type":"object"}`)},
}, "."); err != nil {
t.Fatal(err)
}
endpoint, model := "https://provider.example.test/v1", "fingerprint-model"
required := keyRequired
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: assets,
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: promptkit.ProfileSource("fingerprint profile", map[string]promptkit.ProfileConfig{
"fingerprint-profile": {
Endpoint: &endpoint,
Model: &model,
APIKeyRequired: &required,
ExtraParams: map[string]any{"access_token": parameterValue},
},
}),
}}}}},
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v", err)
}
fingerprints, err := client.LLMCheckpointFingerprints()
if err != nil {
t.Fatalf("LLMCheckpointFingerprints() error = %v", err)
}
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v", fingerprints)
}
return fingerprints[0]
}
baseline := fingerprintFor("original message", "provider-secret-one", false)
if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed {
t.Fatal("captured prompt message change did not alter configuration fingerprint")
}
if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed {
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
}
if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed {
t.Fatal("credential-policy change did not alter configuration fingerprint")
}
if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") {
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
}
}
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
const (
firstEndpoint = "http://localhost:8000/v1"
secondEndpoint = "https://inference.example.test/v1"
)
fingerprintsFor := func(endpoint string, concurrencyLimit int) []CheckpointFingerprint {
fingerprintsFor := func(endpoint string, concurrencyLimit, queueCapacity int) []CheckpointFingerprint {
t.Helper()
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
Source: promptkit.BackendSource("operator scheduling", map[string]promptkit.BackendConfig{
promptkit.BackendLocal: {
Endpoint: promptKitTestString("http://operator-local.test/v1"),
ConcurrencyLimit: promptKitTestInt(1),
QueueCapacity: &queueCapacity,
},
}),
}}}}},
LocalBackend: &PromptKitLocalBackendConfig{
Endpoint: endpoint,
ConcurrencyLimit: concurrencyLimit,
@@ -789,20 +784,22 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
return values
}
baseline := fingerprintsFor(firstEndpoint, 0)
if len(baseline) != 2 ||
baseline[0].Name != promptKitProfileFingerprintName ||
baseline[1].Name != promptKitLocalBackendFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want profile source then local backend target", baseline)
baseline := fingerprintsFor(firstEndpoint, 2, 1)
if len(baseline) != 1 || baseline[0].Name != promptKitConfigurationFingerprintName {
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", baseline)
}
endpointChanged := fingerprintsFor(secondEndpoint, 0)
if baseline[0] != endpointChanged[0] || baseline[1] == endpointChanged[1] {
t.Fatalf("endpoint fingerprints = %#v and %#v, want only local target to change", baseline, endpointChanged)
endpointChanged := fingerprintsFor(secondEndpoint, 2, 1)
if baseline[0] == endpointChanged[0] {
t.Fatalf("configuration fingerprints = %#v and %#v, want endpoint change to affect identity", baseline, endpointChanged)
}
concurrencyChanged := fingerprintsFor(firstEndpoint, 4)
concurrencyChanged := fingerprintsFor(firstEndpoint, 4, 1)
if !reflect.DeepEqual(baseline, concurrencyChanged) {
t.Fatalf("concurrency fingerprints = %#v, want %#v", concurrencyChanged, baseline)
}
queueCapacityChanged := fingerprintsFor(firstEndpoint, 2, 2)
if !reflect.DeepEqual(baseline, queueCapacityChanged) {
t.Fatalf("queue-capacity fingerprints = %#v, want %#v", queueCapacityChanged, baseline)
}
for _, values := range [][]CheckpointFingerprint{baseline, endpointChanged} {
for _, fingerprint := range values {
if strings.Contains(fingerprint.Value, firstEndpoint) ||
@@ -814,7 +811,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
localBackend := &PromptKitLocalBackendConfig{
Endpoint: " " + firstEndpoint + " ",
ConcurrencyLimit: 0,
ConcurrencyLimit: 2,
}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
@@ -831,8 +828,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
if !reflect.DeepEqual(copy, baseline) {
t.Fatalf("fingerprints after input mutation = %#v, want retained target %#v", copy, baseline)
}
copy[0].Value = "mutated-profile"
copy[1].Value = "mutated-target"
copy[0].Value = "mutated-configuration"
fresh, err := client.LLMCheckpointFingerprints()
if err != nil {
t.Fatal(err)