Capture immutable PromptKit configuration identity
This commit is contained in:
@@ -40,13 +40,10 @@ type PromptKitClientConfig struct {
|
||||
}
|
||||
|
||||
type PromptKitClient struct {
|
||||
engine *promptkit.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
profileDir string
|
||||
profileFile string
|
||||
localEndpoint string
|
||||
fallbackProfileDigest string
|
||||
reasoningEffort *string
|
||||
engine *promptkit.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
checkpointFingerprints []CheckpointFingerprint
|
||||
reasoningEffort *string
|
||||
}
|
||||
|
||||
type LLMProfileRecorder struct {
|
||||
@@ -73,7 +70,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
if cfg.Assets == nil {
|
||||
return nil, fmt.Errorf("PromptKit client assets must not be nil")
|
||||
}
|
||||
profileSource, sources, options, err := promptKitEngineConfiguration(PromptKitProfileSourceConfig{
|
||||
_, sources, options, err := promptKitEngineConfiguration(PromptKitProfileSourceConfig{
|
||||
ProfileDir: cfg.ProfileDir,
|
||||
ProfileFile: cfg.ProfileFile,
|
||||
LocalBackend: cfg.LocalBackend,
|
||||
@@ -81,10 +78,6 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
if err != nil {
|
||||
return nil, &promptKitClientConfigurationError{err: err}
|
||||
}
|
||||
fallbackProfileDigest, err := cfg.Assets.FallbackProfileDigest()
|
||||
if err != nil {
|
||||
return nil, &promptKitClientConfigurationError{err: err}
|
||||
}
|
||||
options = append(options, cfg.EngineOptions...)
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
@@ -96,6 +89,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
if err != nil {
|
||||
return nil, &promptKitClientConfigurationError{err: err}
|
||||
}
|
||||
fingerprint, err := promptKitConfigurationFingerprint(engine)
|
||||
if err != nil {
|
||||
return nil, &promptKitClientConfigurationError{err: err}
|
||||
}
|
||||
recorder := cfg.Recorder
|
||||
if recorder == nil {
|
||||
recorder = NewLLMProfileRecorder()
|
||||
@@ -106,13 +103,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
||||
reasoningEffort = &value
|
||||
}
|
||||
return &PromptKitClient{
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
profileDir: profileSource.ProfileDir,
|
||||
profileFile: profileSource.ProfileFile,
|
||||
localEndpoint: profileSource.localEndpoint(),
|
||||
fallbackProfileDigest: fallbackProfileDigest,
|
||||
reasoningEffort: reasoningEffort,
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
checkpointFingerprints: []CheckpointFingerprint{fingerprint},
|
||||
reasoningEffort: reasoningEffort,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -471,15 +465,7 @@ func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint,
|
||||
if c == nil {
|
||||
return nil, nil
|
||||
}
|
||||
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile, c.fallbackProfileDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fingerprints := []CheckpointFingerprint{fingerprint}
|
||||
if c.localEndpoint != "" {
|
||||
fingerprints = append(fingerprints, promptKitLocalBackendFingerprint(c.localEndpoint))
|
||||
}
|
||||
return fingerprints, nil
|
||||
return append([]CheckpointFingerprint(nil), c.checkpointFingerprints...), nil
|
||||
}
|
||||
|
||||
func NewLLMProfileRecorder() *LLMProfileRecorder {
|
||||
|
||||
@@ -401,7 +401,7 @@ func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.
|
||||
})
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *testing.T) {
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
|
||||
writeProfile := func(model string) {
|
||||
@@ -411,7 +411,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
fingerprintFor := func() CheckpointFingerprint {
|
||||
clientFor := func() *PromptKitClient {
|
||||
t.Helper()
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
@@ -420,34 +420,42 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
fingerprintFor := func(client *PromptKitClient) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
values, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values)
|
||||
if len(values) != 1 || values[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", values)
|
||||
}
|
||||
return values[0]
|
||||
}
|
||||
|
||||
writeProfile("model-one")
|
||||
first := fingerprintFor()
|
||||
repeated := fingerprintFor()
|
||||
client := clientFor()
|
||||
first := fingerprintFor(client)
|
||||
repeated := fingerprintFor(clientFor())
|
||||
if first != repeated {
|
||||
t.Fatalf("profile-source fingerprint = %#v then %#v for unchanged source", first, repeated)
|
||||
t.Fatalf("configuration fingerprint = %#v then %#v for unchanged source", first, repeated)
|
||||
}
|
||||
writeProfile("model-two")
|
||||
second := fingerprintFor()
|
||||
if captured := fingerprintFor(client); captured != first {
|
||||
t.Fatalf("existing engine fingerprint after source edit = %#v, want %#v", captured, first)
|
||||
}
|
||||
second := fingerprintFor(clientFor())
|
||||
if first == second {
|
||||
t.Fatalf("profile-source fingerprint = %#v for both profile models", first)
|
||||
t.Fatalf("configuration fingerprint = %#v for both profile models", first)
|
||||
}
|
||||
if strings.TrimSpace(first.Value) == "" {
|
||||
t.Fatal("profile-source fingerprint is empty")
|
||||
t.Fatal("configuration fingerprint is empty")
|
||||
}
|
||||
if strings.Contains(first.Value, profilePath) ||
|
||||
strings.Contains(first.Value, "model-one") ||
|
||||
strings.Contains(first.Value, credentialEnvironment) {
|
||||
t.Fatalf("profile-source fingerprint exposes source details: %#v", first)
|
||||
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
|
||||
}
|
||||
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)})
|
||||
@@ -475,92 +483,12 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(secondBuiltin) != 1 || fresh[0] != secondBuiltin[0] {
|
||||
t.Fatalf("built-in profile fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
|
||||
t.Fatalf("built-in configuration fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
|
||||
}
|
||||
if strings.TrimSpace(fresh[0].Value) == "" {
|
||||
t.Fatal("built-in profile fingerprint is empty")
|
||||
t.Fatal("built-in configuration fingerprint is empty")
|
||||
}
|
||||
|
||||
t.Run("directory layout", func(t *testing.T) {
|
||||
profileDir := t.TempDir()
|
||||
firstPath := filepath.Join(profileDir, "first-profile.yaml")
|
||||
secondPath := filepath.Join(profileDir, "second-profile.yaml")
|
||||
content := []byte("profiles:\n directory-profile:\n endpoint: http://promptkit.test/v1\n model: directory-model\n")
|
||||
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(firstPath, secondPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first == second {
|
||||
t.Fatalf("profile-source fingerprint = %#v after source filename changed", first)
|
||||
}
|
||||
if strings.Contains(first.Value, firstPath) || strings.Contains(second.Value, secondPath) {
|
||||
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inherited parent", func(t *testing.T) {
|
||||
profileDir := t.TempDir()
|
||||
parentPath := filepath.Join(profileDir, "parent.yaml")
|
||||
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
||||
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-one\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(leafPath, []byte("profiles:\n leaf:\n extends: parent\n model: leaf-model\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-two\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
|
||||
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
||||
for _, source := range []struct {
|
||||
name string
|
||||
profileDir string
|
||||
profileFile string
|
||||
}{
|
||||
{
|
||||
name: "file",
|
||||
profileFile: filepath.Join(t.TempDir(), "missing-profile.yaml"),
|
||||
},
|
||||
{
|
||||
name: "directory",
|
||||
profileDir: filepath.Join(t.TempDir(), "missing-profiles"),
|
||||
},
|
||||
} {
|
||||
t.Run(source.name, func(t *testing.T) {
|
||||
_, err := promptKitProfileFingerprint(source.profileDir, source.profileFile, "")
|
||||
if err == nil {
|
||||
t.Fatal("promptKitProfileFingerprint() error = nil, want source read failure")
|
||||
}
|
||||
if (source.profileDir != "" && strings.Contains(err.Error(), source.profileDir)) ||
|
||||
(source.profileFile != "" && strings.Contains(err.Error(), source.profileFile)) {
|
||||
t.Fatalf("fingerprint error exposes profile source path: %q", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
|
||||
@@ -749,8 +677,8 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want captured configuration identity", fingerprints)
|
||||
}
|
||||
return fingerprints[0]
|
||||
}
|
||||
@@ -765,15 +693,82 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
|
||||
fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
assets := NewAssetRegistry()
|
||||
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
|
||||
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||
"prompt.yaml": {Data: []byte(promptDocument)},
|
||||
}, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := assets.RegisterSchemaFS(fstest.MapFS{
|
||||
"result.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoint, model := "https://provider.example.test/v1", "fingerprint-model"
|
||||
required := keyRequired
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||
Source: promptkit.ProfileSource("fingerprint profile", map[string]promptkit.ProfileConfig{
|
||||
"fingerprint-profile": {
|
||||
Endpoint: &endpoint,
|
||||
Model: &model,
|
||||
APIKeyRequired: &required,
|
||||
ExtraParams: map[string]any{"access_token": parameterValue},
|
||||
},
|
||||
}),
|
||||
}}}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||
}
|
||||
fingerprints, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatalf("LLMCheckpointFingerprints() error = %v", err)
|
||||
}
|
||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v", fingerprints)
|
||||
}
|
||||
return fingerprints[0]
|
||||
}
|
||||
|
||||
baseline := fingerprintFor("original message", "provider-secret-one", false)
|
||||
if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed {
|
||||
t.Fatal("captured prompt message change did not alter configuration fingerprint")
|
||||
}
|
||||
if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed {
|
||||
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
|
||||
}
|
||||
if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed {
|
||||
t.Fatal("credential-policy change did not alter configuration fingerprint")
|
||||
}
|
||||
if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") {
|
||||
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
|
||||
const (
|
||||
firstEndpoint = "http://localhost:8000/v1"
|
||||
secondEndpoint = "https://inference.example.test/v1"
|
||||
)
|
||||
fingerprintsFor := func(endpoint string, concurrencyLimit int) []CheckpointFingerprint {
|
||||
fingerprintsFor := func(endpoint string, concurrencyLimit, queueCapacity int) []CheckpointFingerprint {
|
||||
t.Helper()
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||
Source: promptkit.BackendSource("operator scheduling", map[string]promptkit.BackendConfig{
|
||||
promptkit.BackendLocal: {
|
||||
Endpoint: promptKitTestString("http://operator-local.test/v1"),
|
||||
ConcurrencyLimit: promptKitTestInt(1),
|
||||
QueueCapacity: &queueCapacity,
|
||||
},
|
||||
}),
|
||||
}}}}},
|
||||
LocalBackend: &PromptKitLocalBackendConfig{
|
||||
Endpoint: endpoint,
|
||||
ConcurrencyLimit: concurrencyLimit,
|
||||
@@ -789,20 +784,22 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
||||
return values
|
||||
}
|
||||
|
||||
baseline := fingerprintsFor(firstEndpoint, 0)
|
||||
if len(baseline) != 2 ||
|
||||
baseline[0].Name != promptKitProfileFingerprintName ||
|
||||
baseline[1].Name != promptKitLocalBackendFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source then local backend target", baseline)
|
||||
baseline := fingerprintsFor(firstEndpoint, 2, 1)
|
||||
if len(baseline) != 1 || baseline[0].Name != promptKitConfigurationFingerprintName {
|
||||
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", baseline)
|
||||
}
|
||||
endpointChanged := fingerprintsFor(secondEndpoint, 0)
|
||||
if baseline[0] != endpointChanged[0] || baseline[1] == endpointChanged[1] {
|
||||
t.Fatalf("endpoint fingerprints = %#v and %#v, want only local target to change", baseline, endpointChanged)
|
||||
endpointChanged := fingerprintsFor(secondEndpoint, 2, 1)
|
||||
if baseline[0] == endpointChanged[0] {
|
||||
t.Fatalf("configuration fingerprints = %#v and %#v, want endpoint change to affect identity", baseline, endpointChanged)
|
||||
}
|
||||
concurrencyChanged := fingerprintsFor(firstEndpoint, 4)
|
||||
concurrencyChanged := fingerprintsFor(firstEndpoint, 4, 1)
|
||||
if !reflect.DeepEqual(baseline, concurrencyChanged) {
|
||||
t.Fatalf("concurrency fingerprints = %#v, want %#v", concurrencyChanged, baseline)
|
||||
}
|
||||
queueCapacityChanged := fingerprintsFor(firstEndpoint, 2, 2)
|
||||
if !reflect.DeepEqual(baseline, queueCapacityChanged) {
|
||||
t.Fatalf("queue-capacity fingerprints = %#v, want %#v", queueCapacityChanged, baseline)
|
||||
}
|
||||
for _, values := range [][]CheckpointFingerprint{baseline, endpointChanged} {
|
||||
for _, fingerprint := range values {
|
||||
if strings.Contains(fingerprint.Value, firstEndpoint) ||
|
||||
@@ -814,7 +811,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
||||
|
||||
localBackend := &PromptKitLocalBackendConfig{
|
||||
Endpoint: " " + firstEndpoint + " ",
|
||||
ConcurrencyLimit: 0,
|
||||
ConcurrencyLimit: 2,
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
@@ -831,8 +828,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
||||
if !reflect.DeepEqual(copy, baseline) {
|
||||
t.Fatalf("fingerprints after input mutation = %#v, want retained target %#v", copy, baseline)
|
||||
}
|
||||
copy[0].Value = "mutated-profile"
|
||||
copy[1].Value = "mutated-target"
|
||||
copy[0].Value = "mutated-configuration"
|
||||
fresh, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
const promptKitConfigurationFingerprintName = "promptkit_configuration"
|
||||
|
||||
// This version describes the public definition graph captured by the engine.
|
||||
// Include the release and catalog revisions because they contribute built-in
|
||||
// profiles and other effective definitions without appearing in app sources.
|
||||
const promptKitConfigurationSnapshotMarker = "promptkit:v0.12.0:openrouter:v3.20260930.0:rakestrawhome:v3.20260930.0:configuration-snapshot:v1"
|
||||
|
||||
type promptKitConfigurationFingerprintPayload struct {
|
||||
SnapshotMarker string `json:"snapshot_marker"`
|
||||
Configuration map[string]any `json:"configuration"`
|
||||
Profiles []promptKitProfileFingerprintEntry `json:"profiles"`
|
||||
Prompts []promptKitPromptFingerprintEntry `json:"prompts"`
|
||||
}
|
||||
|
||||
type promptKitProfileFingerprintEntry struct {
|
||||
ID string `json:"id"`
|
||||
Target promptkit.ExecutionTarget `json:"target"`
|
||||
APIKeyRequired bool `json:"api_key_required"`
|
||||
}
|
||||
|
||||
type promptKitPromptFingerprintEntry struct {
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
func promptKitConfigurationFingerprint(engine *promptkit.Engine) (CheckpointFingerprint, error) {
|
||||
if engine == nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("PromptKit engine is unavailable for checkpoint identity")
|
||||
}
|
||||
|
||||
inspection, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit configuration for checkpoint identity")
|
||||
}
|
||||
var report bytes.Buffer
|
||||
if err := inspection.WriteEffective(&report); err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("write captured PromptKit configuration for checkpoint identity")
|
||||
}
|
||||
var configuration map[string]any
|
||||
if err := yaml.Unmarshal(report.Bytes(), &configuration); err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("decode captured PromptKit configuration for checkpoint identity")
|
||||
}
|
||||
if configuration == nil {
|
||||
configuration = map[string]any{}
|
||||
}
|
||||
removeLocalSchedulingFields(configuration)
|
||||
|
||||
payload := promptKitConfigurationFingerprintPayload{
|
||||
SnapshotMarker: promptKitConfigurationSnapshotMarker,
|
||||
Configuration: configuration,
|
||||
}
|
||||
profileIDs := sortedMapKeys(configuration["profiles"])
|
||||
payload.Profiles = make([]promptKitProfileFingerprintEntry, 0, len(profileIDs))
|
||||
for _, id := range profileIDs {
|
||||
profile, err := engine.InspectProfile(context.Background(), id)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit profile for checkpoint identity")
|
||||
}
|
||||
payload.Profiles = append(payload.Profiles, promptKitProfileFingerprintEntry{
|
||||
ID: profile.ProfileID,
|
||||
Target: profile.EffectiveModelParams,
|
||||
APIKeyRequired: profile.APIKeyRequired,
|
||||
})
|
||||
}
|
||||
|
||||
promptDefinitions, _ := configuration["prompts"].(map[string]any)
|
||||
var promptIDs []string
|
||||
for id := range promptDefinitions {
|
||||
promptIDs = append(promptIDs, id)
|
||||
}
|
||||
sort.Strings(promptIDs)
|
||||
for _, id := range promptIDs {
|
||||
versions := sortedMapKeys(promptDefinitions[id])
|
||||
for _, version := range versions {
|
||||
prompt, err := engine.InspectPrompt(context.Background(), id, version)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit prompt for checkpoint identity")
|
||||
}
|
||||
payload.Prompts = append(payload.Prompts, promptKitPromptFingerprintEntry{
|
||||
ID: prompt.PromptID,
|
||||
Version: prompt.PromptVersion,
|
||||
Hash: prompt.PromptHash,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("encode captured PromptKit checkpoint identity")
|
||||
}
|
||||
digest := sha256.Sum256(encoded)
|
||||
return CheckpointFingerprint{
|
||||
Name: promptKitConfigurationFingerprintName,
|
||||
Value: "sha256:" + hex.EncodeToString(digest[:]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sortedMapKeys(value any) []string {
|
||||
values, _ := value.(map[string]any)
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func removeLocalSchedulingFields(configuration map[string]any) {
|
||||
backends, _ := configuration["backends"].(map[string]any)
|
||||
local, _ := backends["local"].(map[string]any)
|
||||
delete(local, "concurrency_limit")
|
||||
delete(local, "queue_capacity")
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
promptKitProfileFingerprintName = "promptkit_profile_source"
|
||||
promptKitLocalBackendFingerprintName = "promptkit_local_backend_target"
|
||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
||||
// release. Update this identity when the dependency is upgraded.
|
||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.11.2:builtin-profiles"
|
||||
)
|
||||
|
||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
||||
hasher := sha256.New()
|
||||
writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID))
|
||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(fallbackProfileDigest)))
|
||||
|
||||
switch {
|
||||
case strings.TrimSpace(profileFile) != "":
|
||||
cleanProfileFile := strings.TrimSpace(profileFile)
|
||||
data, err := os.ReadFile(cleanProfileFile)
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, fmt.Errorf("read PromptKit profile file for checkpoint identity")
|
||||
}
|
||||
writeFingerprintPart(hasher, []byte(filepath.ToSlash(filepath.Base(cleanProfileFile))))
|
||||
writeFingerprintPart(hasher, data)
|
||||
case strings.TrimSpace(profileDir) != "":
|
||||
files, err := promptKitProfileFiles(strings.TrimSpace(profileDir))
|
||||
if err != nil {
|
||||
return CheckpointFingerprint{}, err
|
||||
}
|
||||
for _, file := range files {
|
||||
writeFingerprintPart(hasher, []byte(file.path))
|
||||
writeFingerprintPart(hasher, file.digest)
|
||||
}
|
||||
}
|
||||
|
||||
return CheckpointFingerprint{
|
||||
Name: promptKitProfileFingerprintName,
|
||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func promptKitLocalBackendFingerprint(endpoint string) CheckpointFingerprint {
|
||||
hasher := sha256.New()
|
||||
writeFingerprintPart(hasher, []byte(promptKitLocalBackendMarker))
|
||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(endpoint)))
|
||||
return CheckpointFingerprint{
|
||||
Name: promptKitLocalBackendFingerprintName,
|
||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
||||
}
|
||||
}
|
||||
|
||||
type promptKitProfileFile struct {
|
||||
path string
|
||||
digest []byte
|
||||
}
|
||||
|
||||
func promptKitProfileFiles(root string) ([]promptKitProfileFile, error) {
|
||||
var files []promptKitProfileFile
|
||||
err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
extension := filepath.Ext(entry.Name())
|
||||
if extension != ".yaml" && extension != ".yml" {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relativePath, err := filepath.Rel(root, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
files = append(files, promptKitProfileFile{
|
||||
path: filepath.ToSlash(relativePath),
|
||||
digest: append([]byte(nil), sum[:]...),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read PromptKit profile directory for checkpoint identity")
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].path < files[j].path
|
||||
})
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func writeFingerprintPart(hasher interface{ Write([]byte) (int, error) }, value []byte) {
|
||||
length := []byte(fmt.Sprintf("%d:", len(value)))
|
||||
_, _ = hasher.Write(length)
|
||||
_, _ = hasher.Write(value)
|
||||
}
|
||||
Reference in New Issue
Block a user