Protect captured PromptKit checkpoint identity
This commit is contained in:
@@ -435,6 +435,17 @@ Prompt size: one private snapshot/digest mechanism; broader reuse tests follow.
|
||||
|
||||
## Stage 8 — Protect Captured Identity And Resume Behavior
|
||||
|
||||
> Completion: complete. Added edit-after-construction coverage for an imported
|
||||
> inherited profile and its source-relative prompt message: the existing client
|
||||
> retains its digest and execution behavior, while a new client captures both
|
||||
> edits. Fingerprint tests cover model, endpoint, credential policy, redacted
|
||||
> provider parameters, prompt content, null-versus-zero presence, and local
|
||||
> scheduling exclusions without exposing their values. The recorded-checkpoint
|
||||
> test confirms same-snapshot reuse and stale execution misses under the new
|
||||
> `promptkit_configuration` identity; existing override and chunk-plan checks
|
||||
> remain passing. `GOWORK=off go test ./internal/framework/llm ./internal/cli
|
||||
> ./internal/modules/integration` and `git diff --check` pass.
|
||||
|
||||
Goal: prove identity correctness across edits and checkpoint reuse.
|
||||
|
||||
Inspect adapter identity tests, CLI fingerprint handoff, and existing recorded-
|
||||
|
||||
@@ -501,11 +501,11 @@ func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
|
||||
}
|
||||
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
|
||||
if !ok {
|
||||
t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client)
|
||||
t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client)
|
||||
}
|
||||
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
|
||||
if err != nil || len(fingerprints) != 1 {
|
||||
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err)
|
||||
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err)
|
||||
}
|
||||
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
|
||||
t.Fatalf("production LLM client %T does not provide profile manifests", client)
|
||||
@@ -547,7 +547,7 @@ func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *test
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
|
||||
t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
||||
t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -214,7 +214,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
||||
t.Fatal(err)
|
||||
}
|
||||
fingerprints := prepared.CheckpointFingerprints()
|
||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
|
||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}}
|
||||
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
||||
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
||||
if err != nil {
|
||||
@@ -274,16 +274,16 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
||||
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
||||
}
|
||||
|
||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
|
||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}}
|
||||
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
||||
t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
|
||||
t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision)
|
||||
}
|
||||
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||
t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
|
||||
t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -454,6 +454,7 @@ func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *test
|
||||
}
|
||||
if strings.Contains(first.Value, profilePath) ||
|
||||
strings.Contains(first.Value, "model-one") ||
|
||||
strings.Contains(first.Value, "promptkit.test") ||
|
||||
strings.Contains(first.Value, credentialEnvironment) {
|
||||
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
|
||||
}
|
||||
@@ -693,8 +694,153 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientFingerprintAndExecutionRetainImportedSnapshot(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
operatorPath := filepath.Join(directory, "operator.yaml")
|
||||
importedPath := filepath.Join(directory, "inherited.yaml")
|
||||
messagePath := filepath.Join(directory, "message.tmpl")
|
||||
writeFile := func(path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
writeFile(importedPath, `profiles:
|
||||
inherited:
|
||||
endpoint: https://example.invalid/v1
|
||||
model: model-before
|
||||
temperature: 0.7
|
||||
`)
|
||||
writeFile(operatorPath, strings.TrimSpace(`configkit:
|
||||
imports:
|
||||
- path: inherited.yaml
|
||||
profiles:
|
||||
captured:
|
||||
extends: inherited
|
||||
prompts:
|
||||
capture.test:
|
||||
v1:
|
||||
messages:
|
||||
- role: user
|
||||
content_file: message.tmpl
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: schema.json
|
||||
repair_attempts: 0
|
||||
`))
|
||||
writeFile(messagePath, "message before")
|
||||
newClient := func() (*PromptKitClient, *fakePromptKitLLM) {
|
||||
t.Helper()
|
||||
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||
assets := newTestPromptKitAssets(t)
|
||||
if err := assets.RegisterSchemaFS(fstest.MapFS{
|
||||
"schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatalf("RegisterSchemaFS() error = %v", err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
ProfileFile: operatorPath,
|
||||
EngineOptions: []promptkit.Option{
|
||||
promptkit.WithLLMClient(fake),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
var loadErr *configkit.LoadError
|
||||
if errors.As(err, &loadErr) {
|
||||
t.Fatalf("NewPromptKitClient() error = %v (diagnostics %#v)", err, loadErr.Diagnostics())
|
||||
}
|
||||
t.Fatalf("NewPromptKitClient() error = %v (cause %v)", err, errors.Unwrap(err))
|
||||
}
|
||||
return client, fake
|
||||
}
|
||||
fingerprint := func(client *PromptKitClient) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
values, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil || len(values) != 1 {
|
||||
t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", values, err)
|
||||
}
|
||||
return values[0]
|
||||
}
|
||||
complete := func(client *PromptKitClient, fake *fakePromptKitLLM) (string, string) {
|
||||
t.Helper()
|
||||
var output map[string]any
|
||||
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "capture.test", ProfileID: "captured", SessionID: "captured-snapshot",
|
||||
}, &output)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured() error = %v", err)
|
||||
}
|
||||
request := fake.lastRequest()
|
||||
if len(request.Prompt.Messages) != 1 {
|
||||
t.Fatalf("rendered messages = %#v", request.Prompt.Messages)
|
||||
}
|
||||
return request.Target.Model, request.Prompt.Messages[0].Content
|
||||
}
|
||||
|
||||
first, firstFake := newClient()
|
||||
firstFingerprint := fingerprint(first)
|
||||
if model, body := complete(first, firstFake); model != "model-before" || body != "message before" {
|
||||
t.Fatalf("initial execution model=%q body=%q", model, body)
|
||||
}
|
||||
writeFile(importedPath, `profiles:
|
||||
inherited:
|
||||
endpoint: https://example.invalid/v1
|
||||
model: model-after
|
||||
temperature: 0.7
|
||||
`)
|
||||
writeFile(messagePath, "message after")
|
||||
if got := fingerprint(first); got != firstFingerprint {
|
||||
t.Fatalf("existing engine fingerprint after source edits = %#v, want %#v", got, firstFingerprint)
|
||||
}
|
||||
if model, body := complete(first, firstFake); model != "model-before" || body != "message before" {
|
||||
t.Fatalf("existing engine execution changed after source edits: model=%q body=%q", model, body)
|
||||
}
|
||||
second, secondFake := newClient()
|
||||
if got := fingerprint(second); got == firstFingerprint {
|
||||
t.Fatalf("new engine fingerprint = %#v, want changed imported definition/message identity", got)
|
||||
}
|
||||
if model, body := complete(second, secondFake); model != "model-after" || body != "message after" {
|
||||
t.Fatalf("new engine execution model=%q body=%q, want edited snapshot", model, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitClientFingerprintPreservesNullVersusZero(t *testing.T) {
|
||||
fingerprintFor := func(temperature string) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||
content := `profiles:
|
||||
parent:
|
||||
endpoint: https://example.invalid/v1
|
||||
model: presence-model
|
||||
temperature: 0.7
|
||||
selected:
|
||||
extends: parent
|
||||
temperature: ` + temperature + "\n"
|
||||
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: newTestPromptKitAssets(t),
|
||||
ProfileFile: profilePath,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||
}
|
||||
fingerprints, err := client.LLMCheckpointFingerprints()
|
||||
if err != nil || len(fingerprints) != 1 {
|
||||
t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", fingerprints, err)
|
||||
}
|
||||
return fingerprints[0]
|
||||
}
|
||||
if zero, explicitNull := fingerprintFor("0"), fingerprintFor("null"); zero == explicitNull {
|
||||
t.Fatalf("null and zero profile values share checkpoint identity: %#v", zero)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
|
||||
fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint {
|
||||
fingerprintFor := func(message, endpoint, parameterValue string, keyRequired bool) CheckpointFingerprint {
|
||||
t.Helper()
|
||||
assets := NewAssetRegistry()
|
||||
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
|
||||
@@ -708,7 +854,7 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi
|
||||
}, "."); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoint, model := "https://provider.example.test/v1", "fingerprint-model"
|
||||
model := "fingerprint-model"
|
||||
required := keyRequired
|
||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||
Assets: assets,
|
||||
@@ -736,18 +882,23 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi
|
||||
return fingerprints[0]
|
||||
}
|
||||
|
||||
baseline := fingerprintFor("original message", "provider-secret-one", false)
|
||||
if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed {
|
||||
baseline := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", false)
|
||||
if changed := fingerprintFor("edited message", "https://provider.example.test/v1", "provider-secret-one", false); baseline == changed {
|
||||
t.Fatal("captured prompt message change did not alter configuration fingerprint")
|
||||
}
|
||||
if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed {
|
||||
if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-two", false); baseline == changed {
|
||||
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
|
||||
}
|
||||
if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed {
|
||||
if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", true); baseline == changed {
|
||||
t.Fatal("credential-policy change did not alter configuration fingerprint")
|
||||
}
|
||||
if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") {
|
||||
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
|
||||
if changed := fingerprintFor("original message", "https://other.example.test/v1", "provider-secret-one", false); baseline == changed {
|
||||
t.Fatal("profile endpoint change did not alter configuration fingerprint")
|
||||
}
|
||||
for _, rawValue := range []string{"provider-secret-one", "original message", "provider.example.test"} {
|
||||
if strings.Contains(baseline.Value, rawValue) {
|
||||
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user