Capture immutable PromptKit configuration identity
This commit is contained in:
@@ -398,6 +398,14 @@ Prompt size: operator-source fixtures, not identity or content lifecycle.
|
|||||||
|
|
||||||
## Stage 7 — Compute Immutable Captured Configuration Identity
|
## Stage 7 — Compute Immutable Captured Configuration Identity
|
||||||
|
|
||||||
|
> Completion: complete. Checkpoint identity is now a single digest captured
|
||||||
|
> from each constructed engine's effective configuration, profile targets,
|
||||||
|
> credential policy, and prompt hashes. Focused tests confirm source edits do
|
||||||
|
> not mutate an existing engine's identity, provider parameters and prompt
|
||||||
|
> bodies affect identity without being exposed, and local scheduling values
|
||||||
|
> remain excluded. `GOWORK=off go test ./internal/framework/llm`,
|
||||||
|
> `GOWORK=off go mod tidy -diff`, and `git diff --check` pass.
|
||||||
|
|
||||||
Goal: replace live-file fingerprinting with the settled snapshot algorithm.
|
Goal: replace live-file fingerprinting with the settled snapshot algorithm.
|
||||||
|
|
||||||
Inspect `promptkit_profile_fingerprint.go`, client construction/fingerprint
|
Inspect `promptkit_profile_fingerprint.go`, client construction/fingerprint
|
||||||
|
|||||||
6
go.mod
6
go.mod
@@ -8,10 +8,12 @@ require (
|
|||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
)
|
)
|
||||||
|
|
||||||
require golang.org/x/text v0.40.0
|
require (
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0-rc.6
|
||||||
|
golang.org/x/text v0.40.0
|
||||||
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -40,13 +40,10 @@ type PromptKitClientConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PromptKitClient struct {
|
type PromptKitClient struct {
|
||||||
engine *promptkit.Engine
|
engine *promptkit.Engine
|
||||||
recorder *LLMProfileRecorder
|
recorder *LLMProfileRecorder
|
||||||
profileDir string
|
checkpointFingerprints []CheckpointFingerprint
|
||||||
profileFile string
|
reasoningEffort *string
|
||||||
localEndpoint string
|
|
||||||
fallbackProfileDigest string
|
|
||||||
reasoningEffort *string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type LLMProfileRecorder struct {
|
type LLMProfileRecorder struct {
|
||||||
@@ -73,7 +70,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
|||||||
if cfg.Assets == nil {
|
if cfg.Assets == nil {
|
||||||
return nil, fmt.Errorf("PromptKit client assets must not be nil")
|
return nil, fmt.Errorf("PromptKit client assets must not be nil")
|
||||||
}
|
}
|
||||||
profileSource, sources, options, err := promptKitEngineConfiguration(PromptKitProfileSourceConfig{
|
_, sources, options, err := promptKitEngineConfiguration(PromptKitProfileSourceConfig{
|
||||||
ProfileDir: cfg.ProfileDir,
|
ProfileDir: cfg.ProfileDir,
|
||||||
ProfileFile: cfg.ProfileFile,
|
ProfileFile: cfg.ProfileFile,
|
||||||
LocalBackend: cfg.LocalBackend,
|
LocalBackend: cfg.LocalBackend,
|
||||||
@@ -81,10 +78,6 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &promptKitClientConfigurationError{err: err}
|
return nil, &promptKitClientConfigurationError{err: err}
|
||||||
}
|
}
|
||||||
fallbackProfileDigest, err := cfg.Assets.FallbackProfileDigest()
|
|
||||||
if err != nil {
|
|
||||||
return nil, &promptKitClientConfigurationError{err: err}
|
|
||||||
}
|
|
||||||
options = append(options, cfg.EngineOptions...)
|
options = append(options, cfg.EngineOptions...)
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
@@ -96,6 +89,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, &promptKitClientConfigurationError{err: err}
|
return nil, &promptKitClientConfigurationError{err: err}
|
||||||
}
|
}
|
||||||
|
fingerprint, err := promptKitConfigurationFingerprint(engine)
|
||||||
|
if err != nil {
|
||||||
|
return nil, &promptKitClientConfigurationError{err: err}
|
||||||
|
}
|
||||||
recorder := cfg.Recorder
|
recorder := cfg.Recorder
|
||||||
if recorder == nil {
|
if recorder == nil {
|
||||||
recorder = NewLLMProfileRecorder()
|
recorder = NewLLMProfileRecorder()
|
||||||
@@ -106,13 +103,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
|
|||||||
reasoningEffort = &value
|
reasoningEffort = &value
|
||||||
}
|
}
|
||||||
return &PromptKitClient{
|
return &PromptKitClient{
|
||||||
engine: engine,
|
engine: engine,
|
||||||
recorder: recorder,
|
recorder: recorder,
|
||||||
profileDir: profileSource.ProfileDir,
|
checkpointFingerprints: []CheckpointFingerprint{fingerprint},
|
||||||
profileFile: profileSource.ProfileFile,
|
reasoningEffort: reasoningEffort,
|
||||||
localEndpoint: profileSource.localEndpoint(),
|
|
||||||
fallbackProfileDigest: fallbackProfileDigest,
|
|
||||||
reasoningEffort: reasoningEffort,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -471,15 +465,7 @@ func (c *PromptKitClient) LLMCheckpointFingerprints() ([]CheckpointFingerprint,
|
|||||||
if c == nil {
|
if c == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
fingerprint, err := promptKitProfileFingerprint(c.profileDir, c.profileFile, c.fallbackProfileDigest)
|
return append([]CheckpointFingerprint(nil), c.checkpointFingerprints...), nil
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
fingerprints := []CheckpointFingerprint{fingerprint}
|
|
||||||
if c.localEndpoint != "" {
|
|
||||||
fingerprints = append(fingerprints, promptKitLocalBackendFingerprint(c.localEndpoint))
|
|
||||||
}
|
|
||||||
return fingerprints, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewLLMProfileRecorder() *LLMProfileRecorder {
|
func NewLLMProfileRecorder() *LLMProfileRecorder {
|
||||||
|
|||||||
@@ -401,7 +401,7 @@ func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *testing.T) {
|
||||||
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
|
||||||
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
|
const credentialEnvironment = "PROMPTKIT_TEST_API_KEY"
|
||||||
writeProfile := func(model string) {
|
writeProfile := func(model string) {
|
||||||
@@ -411,7 +411,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
fingerprintFor := func() CheckpointFingerprint {
|
clientFor := func() *PromptKitClient {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
@@ -420,34 +420,42 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
fingerprintFor := func(client *PromptKitClient) CheckpointFingerprint {
|
||||||
|
t.Helper()
|
||||||
values, err := client.LLMCheckpointFingerprints()
|
values, err := client.LLMCheckpointFingerprints()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(values) != 1 || values[0].Name != promptKitProfileFingerprintName {
|
if len(values) != 1 || values[0].Name != promptKitConfigurationFingerprintName {
|
||||||
t.Fatalf("checkpoint fingerprints = %#v, want one profile-source identity", values)
|
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", values)
|
||||||
}
|
}
|
||||||
return values[0]
|
return values[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
writeProfile("model-one")
|
writeProfile("model-one")
|
||||||
first := fingerprintFor()
|
client := clientFor()
|
||||||
repeated := fingerprintFor()
|
first := fingerprintFor(client)
|
||||||
|
repeated := fingerprintFor(clientFor())
|
||||||
if first != repeated {
|
if first != repeated {
|
||||||
t.Fatalf("profile-source fingerprint = %#v then %#v for unchanged source", first, repeated)
|
t.Fatalf("configuration fingerprint = %#v then %#v for unchanged source", first, repeated)
|
||||||
}
|
}
|
||||||
writeProfile("model-two")
|
writeProfile("model-two")
|
||||||
second := fingerprintFor()
|
if captured := fingerprintFor(client); captured != first {
|
||||||
|
t.Fatalf("existing engine fingerprint after source edit = %#v, want %#v", captured, first)
|
||||||
|
}
|
||||||
|
second := fingerprintFor(clientFor())
|
||||||
if first == second {
|
if first == second {
|
||||||
t.Fatalf("profile-source fingerprint = %#v for both profile models", first)
|
t.Fatalf("configuration fingerprint = %#v for both profile models", first)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(first.Value) == "" {
|
if strings.TrimSpace(first.Value) == "" {
|
||||||
t.Fatal("profile-source fingerprint is empty")
|
t.Fatal("configuration fingerprint is empty")
|
||||||
}
|
}
|
||||||
if strings.Contains(first.Value, profilePath) ||
|
if strings.Contains(first.Value, profilePath) ||
|
||||||
strings.Contains(first.Value, "model-one") ||
|
strings.Contains(first.Value, "model-one") ||
|
||||||
strings.Contains(first.Value, credentialEnvironment) {
|
strings.Contains(first.Value, credentialEnvironment) {
|
||||||
t.Fatalf("profile-source fingerprint exposes source details: %#v", first)
|
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)})
|
client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t)})
|
||||||
@@ -475,92 +483,12 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(secondBuiltin) != 1 || fresh[0] != secondBuiltin[0] {
|
if len(secondBuiltin) != 1 || fresh[0] != secondBuiltin[0] {
|
||||||
t.Fatalf("built-in profile fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
|
t.Fatalf("built-in configuration fingerprints = %#v and %#v, want deterministic identity", fresh, secondBuiltin)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(fresh[0].Value) == "" {
|
if strings.TrimSpace(fresh[0].Value) == "" {
|
||||||
t.Fatal("built-in profile fingerprint is empty")
|
t.Fatal("built-in configuration fingerprint is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("directory layout", func(t *testing.T) {
|
|
||||||
profileDir := t.TempDir()
|
|
||||||
firstPath := filepath.Join(profileDir, "first-profile.yaml")
|
|
||||||
secondPath := filepath.Join(profileDir, "second-profile.yaml")
|
|
||||||
content := []byte("profiles:\n directory-profile:\n endpoint: http://promptkit.test/v1\n model: directory-model\n")
|
|
||||||
if err := os.WriteFile(firstPath, content, 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.Rename(firstPath, secondPath); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if first == second {
|
|
||||||
t.Fatalf("profile-source fingerprint = %#v after source filename changed", first)
|
|
||||||
}
|
|
||||||
if strings.Contains(first.Value, firstPath) || strings.Contains(second.Value, secondPath) {
|
|
||||||
t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("inherited parent", func(t *testing.T) {
|
|
||||||
profileDir := t.TempDir()
|
|
||||||
parentPath := filepath.Join(profileDir, "parent.yaml")
|
|
||||||
leafPath := filepath.Join(profileDir, "leaf.yaml")
|
|
||||||
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-one\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(leafPath, []byte("profiles:\n leaf:\n extends: parent\n model: leaf-model\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
first, err := promptKitProfileFingerprint(profileDir, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-two\n"), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
second, err := promptKitProfileFingerprint(profileDir, "", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if first == second || strings.Contains(first.Value, "parent-one") || strings.Contains(first.Value, parentPath) || strings.Contains(first.Value, leafPath) {
|
|
||||||
t.Fatalf("inherited profile fingerprint = %#v then %#v", first, second)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {
|
|
||||||
for _, source := range []struct {
|
|
||||||
name string
|
|
||||||
profileDir string
|
|
||||||
profileFile string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "file",
|
|
||||||
profileFile: filepath.Join(t.TempDir(), "missing-profile.yaml"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "directory",
|
|
||||||
profileDir: filepath.Join(t.TempDir(), "missing-profiles"),
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(source.name, func(t *testing.T) {
|
|
||||||
_, err := promptKitProfileFingerprint(source.profileDir, source.profileFile, "")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("promptKitProfileFingerprint() error = nil, want source read failure")
|
|
||||||
}
|
|
||||||
if (source.profileDir != "" && strings.Contains(err.Error(), source.profileDir)) ||
|
|
||||||
(source.profileFile != "" && strings.Contains(err.Error(), source.profileFile)) {
|
|
||||||
t.Fatalf("fingerprint error exposes profile source path: %q", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
|
func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing.T) {
|
||||||
@@ -749,8 +677,8 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitProfileFingerprintName {
|
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
|
||||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source identity", fingerprints)
|
t.Fatalf("checkpoint fingerprints = %#v, want captured configuration identity", fingerprints)
|
||||||
}
|
}
|
||||||
return fingerprints[0]
|
return fingerprints[0]
|
||||||
}
|
}
|
||||||
@@ -765,15 +693,82 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
|
||||||
|
fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint {
|
||||||
|
t.Helper()
|
||||||
|
assets := NewAssetRegistry()
|
||||||
|
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
|
||||||
|
if err := assets.RegisterPromptFS(fstest.MapFS{
|
||||||
|
"prompt.yaml": {Data: []byte(promptDocument)},
|
||||||
|
}, "."); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := assets.RegisterSchemaFS(fstest.MapFS{
|
||||||
|
"result.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
|
}, "."); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
endpoint, model := "https://provider.example.test/v1", "fingerprint-model"
|
||||||
|
required := keyRequired
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
|
Assets: assets,
|
||||||
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||||
|
Source: promptkit.ProfileSource("fingerprint profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"fingerprint-profile": {
|
||||||
|
Endpoint: &endpoint,
|
||||||
|
Model: &model,
|
||||||
|
APIKeyRequired: &required,
|
||||||
|
ExtraParams: map[string]any{"access_token": parameterValue},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}}}}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||||
|
}
|
||||||
|
fingerprints, err := client.LLMCheckpointFingerprints()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LLMCheckpointFingerprints() error = %v", err)
|
||||||
|
}
|
||||||
|
if len(fingerprints) != 1 || fingerprints[0].Name != promptKitConfigurationFingerprintName {
|
||||||
|
t.Fatalf("checkpoint fingerprints = %#v", fingerprints)
|
||||||
|
}
|
||||||
|
return fingerprints[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
baseline := fingerprintFor("original message", "provider-secret-one", false)
|
||||||
|
if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed {
|
||||||
|
t.Fatal("captured prompt message change did not alter configuration fingerprint")
|
||||||
|
}
|
||||||
|
if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed {
|
||||||
|
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
|
||||||
|
}
|
||||||
|
if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed {
|
||||||
|
t.Fatal("credential-policy change did not alter configuration fingerprint")
|
||||||
|
}
|
||||||
|
if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") {
|
||||||
|
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
|
func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing.T) {
|
||||||
const (
|
const (
|
||||||
firstEndpoint = "http://localhost:8000/v1"
|
firstEndpoint = "http://localhost:8000/v1"
|
||||||
secondEndpoint = "https://inference.example.test/v1"
|
secondEndpoint = "https://inference.example.test/v1"
|
||||||
)
|
)
|
||||||
fingerprintsFor := func(endpoint string, concurrencyLimit int) []CheckpointFingerprint {
|
fingerprintsFor := func(endpoint string, concurrencyLimit, queueCapacity int) []CheckpointFingerprint {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
|
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{
|
||||||
|
Source: promptkit.BackendSource("operator scheduling", map[string]promptkit.BackendConfig{
|
||||||
|
promptkit.BackendLocal: {
|
||||||
|
Endpoint: promptKitTestString("http://operator-local.test/v1"),
|
||||||
|
ConcurrencyLimit: promptKitTestInt(1),
|
||||||
|
QueueCapacity: &queueCapacity,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}}}}},
|
||||||
LocalBackend: &PromptKitLocalBackendConfig{
|
LocalBackend: &PromptKitLocalBackendConfig{
|
||||||
Endpoint: endpoint,
|
Endpoint: endpoint,
|
||||||
ConcurrencyLimit: concurrencyLimit,
|
ConcurrencyLimit: concurrencyLimit,
|
||||||
@@ -789,20 +784,22 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
|||||||
return values
|
return values
|
||||||
}
|
}
|
||||||
|
|
||||||
baseline := fingerprintsFor(firstEndpoint, 0)
|
baseline := fingerprintsFor(firstEndpoint, 2, 1)
|
||||||
if len(baseline) != 2 ||
|
if len(baseline) != 1 || baseline[0].Name != promptKitConfigurationFingerprintName {
|
||||||
baseline[0].Name != promptKitProfileFingerprintName ||
|
t.Fatalf("checkpoint fingerprints = %#v, want one captured configuration identity", baseline)
|
||||||
baseline[1].Name != promptKitLocalBackendFingerprintName {
|
|
||||||
t.Fatalf("checkpoint fingerprints = %#v, want profile source then local backend target", baseline)
|
|
||||||
}
|
}
|
||||||
endpointChanged := fingerprintsFor(secondEndpoint, 0)
|
endpointChanged := fingerprintsFor(secondEndpoint, 2, 1)
|
||||||
if baseline[0] != endpointChanged[0] || baseline[1] == endpointChanged[1] {
|
if baseline[0] == endpointChanged[0] {
|
||||||
t.Fatalf("endpoint fingerprints = %#v and %#v, want only local target to change", baseline, endpointChanged)
|
t.Fatalf("configuration fingerprints = %#v and %#v, want endpoint change to affect identity", baseline, endpointChanged)
|
||||||
}
|
}
|
||||||
concurrencyChanged := fingerprintsFor(firstEndpoint, 4)
|
concurrencyChanged := fingerprintsFor(firstEndpoint, 4, 1)
|
||||||
if !reflect.DeepEqual(baseline, concurrencyChanged) {
|
if !reflect.DeepEqual(baseline, concurrencyChanged) {
|
||||||
t.Fatalf("concurrency fingerprints = %#v, want %#v", concurrencyChanged, baseline)
|
t.Fatalf("concurrency fingerprints = %#v, want %#v", concurrencyChanged, baseline)
|
||||||
}
|
}
|
||||||
|
queueCapacityChanged := fingerprintsFor(firstEndpoint, 2, 2)
|
||||||
|
if !reflect.DeepEqual(baseline, queueCapacityChanged) {
|
||||||
|
t.Fatalf("queue-capacity fingerprints = %#v, want %#v", queueCapacityChanged, baseline)
|
||||||
|
}
|
||||||
for _, values := range [][]CheckpointFingerprint{baseline, endpointChanged} {
|
for _, values := range [][]CheckpointFingerprint{baseline, endpointChanged} {
|
||||||
for _, fingerprint := range values {
|
for _, fingerprint := range values {
|
||||||
if strings.Contains(fingerprint.Value, firstEndpoint) ||
|
if strings.Contains(fingerprint.Value, firstEndpoint) ||
|
||||||
@@ -814,7 +811,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
|||||||
|
|
||||||
localBackend := &PromptKitLocalBackendConfig{
|
localBackend := &PromptKitLocalBackendConfig{
|
||||||
Endpoint: " " + firstEndpoint + " ",
|
Endpoint: " " + firstEndpoint + " ",
|
||||||
ConcurrencyLimit: 0,
|
ConcurrencyLimit: 2,
|
||||||
}
|
}
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: newTestPromptKitAssets(t),
|
Assets: newTestPromptKitAssets(t),
|
||||||
@@ -831,8 +828,7 @@ func TestPromptKitClientCheckpointFingerprintTracksLocalBackendTarget(t *testing
|
|||||||
if !reflect.DeepEqual(copy, baseline) {
|
if !reflect.DeepEqual(copy, baseline) {
|
||||||
t.Fatalf("fingerprints after input mutation = %#v, want retained target %#v", copy, baseline)
|
t.Fatalf("fingerprints after input mutation = %#v, want retained target %#v", copy, baseline)
|
||||||
}
|
}
|
||||||
copy[0].Value = "mutated-profile"
|
copy[0].Value = "mutated-configuration"
|
||||||
copy[1].Value = "mutated-target"
|
|
||||||
fresh, err := client.LLMCheckpointFingerprints()
|
fresh, err := client.LLMCheckpointFingerprints()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
129
internal/framework/llm/promptkit_configuration_fingerprint.go
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
"go.yaml.in/yaml/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
const promptKitConfigurationFingerprintName = "promptkit_configuration"
|
||||||
|
|
||||||
|
// This version describes the public definition graph captured by the engine.
|
||||||
|
// Include the release and catalog revisions because they contribute built-in
|
||||||
|
// profiles and other effective definitions without appearing in app sources.
|
||||||
|
const promptKitConfigurationSnapshotMarker = "promptkit:v0.12.0:openrouter:v3.20260930.0:rakestrawhome:v3.20260930.0:configuration-snapshot:v1"
|
||||||
|
|
||||||
|
type promptKitConfigurationFingerprintPayload struct {
|
||||||
|
SnapshotMarker string `json:"snapshot_marker"`
|
||||||
|
Configuration map[string]any `json:"configuration"`
|
||||||
|
Profiles []promptKitProfileFingerprintEntry `json:"profiles"`
|
||||||
|
Prompts []promptKitPromptFingerprintEntry `json:"prompts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptKitProfileFingerprintEntry struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Target promptkit.ExecutionTarget `json:"target"`
|
||||||
|
APIKeyRequired bool `json:"api_key_required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type promptKitPromptFingerprintEntry struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
Hash string `json:"hash"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func promptKitConfigurationFingerprint(engine *promptkit.Engine) (CheckpointFingerprint, error) {
|
||||||
|
if engine == nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("PromptKit engine is unavailable for checkpoint identity")
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit configuration for checkpoint identity")
|
||||||
|
}
|
||||||
|
var report bytes.Buffer
|
||||||
|
if err := inspection.WriteEffective(&report); err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("write captured PromptKit configuration for checkpoint identity")
|
||||||
|
}
|
||||||
|
var configuration map[string]any
|
||||||
|
if err := yaml.Unmarshal(report.Bytes(), &configuration); err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("decode captured PromptKit configuration for checkpoint identity")
|
||||||
|
}
|
||||||
|
if configuration == nil {
|
||||||
|
configuration = map[string]any{}
|
||||||
|
}
|
||||||
|
removeLocalSchedulingFields(configuration)
|
||||||
|
|
||||||
|
payload := promptKitConfigurationFingerprintPayload{
|
||||||
|
SnapshotMarker: promptKitConfigurationSnapshotMarker,
|
||||||
|
Configuration: configuration,
|
||||||
|
}
|
||||||
|
profileIDs := sortedMapKeys(configuration["profiles"])
|
||||||
|
payload.Profiles = make([]promptKitProfileFingerprintEntry, 0, len(profileIDs))
|
||||||
|
for _, id := range profileIDs {
|
||||||
|
profile, err := engine.InspectProfile(context.Background(), id)
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit profile for checkpoint identity")
|
||||||
|
}
|
||||||
|
payload.Profiles = append(payload.Profiles, promptKitProfileFingerprintEntry{
|
||||||
|
ID: profile.ProfileID,
|
||||||
|
Target: profile.EffectiveModelParams,
|
||||||
|
APIKeyRequired: profile.APIKeyRequired,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
promptDefinitions, _ := configuration["prompts"].(map[string]any)
|
||||||
|
var promptIDs []string
|
||||||
|
for id := range promptDefinitions {
|
||||||
|
promptIDs = append(promptIDs, id)
|
||||||
|
}
|
||||||
|
sort.Strings(promptIDs)
|
||||||
|
for _, id := range promptIDs {
|
||||||
|
versions := sortedMapKeys(promptDefinitions[id])
|
||||||
|
for _, version := range versions {
|
||||||
|
prompt, err := engine.InspectPrompt(context.Background(), id, version)
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("inspect captured PromptKit prompt for checkpoint identity")
|
||||||
|
}
|
||||||
|
payload.Prompts = append(payload.Prompts, promptKitPromptFingerprintEntry{
|
||||||
|
ID: prompt.PromptID,
|
||||||
|
Version: prompt.PromptVersion,
|
||||||
|
Hash: prompt.PromptHash,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return CheckpointFingerprint{}, fmt.Errorf("encode captured PromptKit checkpoint identity")
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(encoded)
|
||||||
|
return CheckpointFingerprint{
|
||||||
|
Name: promptKitConfigurationFingerprintName,
|
||||||
|
Value: "sha256:" + hex.EncodeToString(digest[:]),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedMapKeys(value any) []string {
|
||||||
|
values, _ := value.(map[string]any)
|
||||||
|
keys := make([]string, 0, len(values))
|
||||||
|
for key := range values {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
return keys
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeLocalSchedulingFields(configuration map[string]any) {
|
||||||
|
backends, _ := configuration["backends"].(map[string]any)
|
||||||
|
local, _ := backends["local"].(map[string]any)
|
||||||
|
delete(local, "concurrency_limit")
|
||||||
|
delete(local, "queue_capacity")
|
||||||
|
}
|
||||||
@@ -1,110 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
promptKitProfileFingerprintName = "promptkit_profile_source"
|
|
||||||
promptKitLocalBackendFingerprintName = "promptkit_local_backend_target"
|
|
||||||
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
|
|
||||||
// The built-in profile catalog is compiled into this pinned PromptKit
|
|
||||||
// release. Update this identity when the dependency is upgraded.
|
|
||||||
promptKitBuiltinProfileCatalogID = "promptkit:v0.11.2:builtin-profiles"
|
|
||||||
)
|
|
||||||
|
|
||||||
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {
|
|
||||||
hasher := sha256.New()
|
|
||||||
writeFingerprintPart(hasher, []byte(promptKitBuiltinProfileCatalogID))
|
|
||||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(fallbackProfileDigest)))
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case strings.TrimSpace(profileFile) != "":
|
|
||||||
cleanProfileFile := strings.TrimSpace(profileFile)
|
|
||||||
data, err := os.ReadFile(cleanProfileFile)
|
|
||||||
if err != nil {
|
|
||||||
return CheckpointFingerprint{}, fmt.Errorf("read PromptKit profile file for checkpoint identity")
|
|
||||||
}
|
|
||||||
writeFingerprintPart(hasher, []byte(filepath.ToSlash(filepath.Base(cleanProfileFile))))
|
|
||||||
writeFingerprintPart(hasher, data)
|
|
||||||
case strings.TrimSpace(profileDir) != "":
|
|
||||||
files, err := promptKitProfileFiles(strings.TrimSpace(profileDir))
|
|
||||||
if err != nil {
|
|
||||||
return CheckpointFingerprint{}, err
|
|
||||||
}
|
|
||||||
for _, file := range files {
|
|
||||||
writeFingerprintPart(hasher, []byte(file.path))
|
|
||||||
writeFingerprintPart(hasher, file.digest)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return CheckpointFingerprint{
|
|
||||||
Name: promptKitProfileFingerprintName,
|
|
||||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptKitLocalBackendFingerprint(endpoint string) CheckpointFingerprint {
|
|
||||||
hasher := sha256.New()
|
|
||||||
writeFingerprintPart(hasher, []byte(promptKitLocalBackendMarker))
|
|
||||||
writeFingerprintPart(hasher, []byte(strings.TrimSpace(endpoint)))
|
|
||||||
return CheckpointFingerprint{
|
|
||||||
Name: promptKitLocalBackendFingerprintName,
|
|
||||||
Value: "sha256:" + hex.EncodeToString(hasher.Sum(nil)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptKitProfileFile struct {
|
|
||||||
path string
|
|
||||||
digest []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptKitProfileFiles(root string) ([]promptKitProfileFile, error) {
|
|
||||||
var files []promptKitProfileFile
|
|
||||||
err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error {
|
|
||||||
if walkErr != nil {
|
|
||||||
return walkErr
|
|
||||||
}
|
|
||||||
if entry.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
extension := filepath.Ext(entry.Name())
|
|
||||||
if extension != ".yaml" && extension != ".yml" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
data, err := os.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
relativePath, err := filepath.Rel(root, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
sum := sha256.Sum256(data)
|
|
||||||
files = append(files, promptKitProfileFile{
|
|
||||||
path: filepath.ToSlash(relativePath),
|
|
||||||
digest: append([]byte(nil), sum[:]...),
|
|
||||||
})
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read PromptKit profile directory for checkpoint identity")
|
|
||||||
}
|
|
||||||
sort.Slice(files, func(i, j int) bool {
|
|
||||||
return files[i].path < files[j].path
|
|
||||||
})
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeFingerprintPart(hasher interface{ Write([]byte) (int, error) }, value []byte) {
|
|
||||||
length := []byte(fmt.Sprintf("%d:", len(value)))
|
|
||||||
_, _ = hasher.Write(length)
|
|
||||||
_, _ = hasher.Write(value)
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user