Protect captured PromptKit checkpoint identity

This commit is contained in:
2026-10-04 13:08:56 +00:00
parent deaa3560bc
commit 7ea9fe12b9
4 changed files with 177 additions and 15 deletions

View File

@@ -435,6 +435,17 @@ Prompt size: one private snapshot/digest mechanism; broader reuse tests follow.
## Stage 8 — Protect Captured Identity And Resume Behavior
> Completion: complete. Added edit-after-construction coverage for an imported
> inherited profile and its source-relative prompt message: the existing client
> retains its digest and execution behavior, while a new client captures both
> edits. Fingerprint tests cover model, endpoint, credential policy, redacted
> provider parameters, prompt content, null-versus-zero presence, and local
> scheduling exclusions without exposing their values. The recorded-checkpoint
> test confirms same-snapshot reuse and stale execution misses under the new
> `promptkit_configuration` identity; existing override and chunk-plan checks
> remain passing. `GOWORK=off go test ./internal/framework/llm ./internal/cli
> ./internal/modules/integration` and `git diff --check` pass.
Goal: prove identity correctness across edits and checkpoint reuse.
Inspect adapter identity tests, CLI fingerprint handoff, and existing recorded-

View File

@@ -501,11 +501,11 @@ func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
}
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
if !ok {
t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client)
t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client)
}
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
if err != nil || len(fingerprints) != 1 {
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err)
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err)
}
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
t.Fatalf("production LLM client %T does not provide profile manifests", client)
@@ -547,7 +547,7 @@ func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *test
t.Fatal(err)
}
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
}
}

View File

@@ -214,7 +214,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
t.Fatal(err)
}
fingerprints := prepared.CheckpointFingerprints()
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}}
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
if err != nil {
@@ -274,16 +274,16 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
}
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}}
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
if err != nil {
t.Fatal(err)
}
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision)
}
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision)
}
}

View File

@@ -454,6 +454,7 @@ func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *test
}
if strings.Contains(first.Value, profilePath) ||
strings.Contains(first.Value, "model-one") ||
strings.Contains(first.Value, "promptkit.test") ||
strings.Contains(first.Value, credentialEnvironment) {
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
}
@@ -693,8 +694,153 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
}
}
func TestPromptKitClientFingerprintAndExecutionRetainImportedSnapshot(t *testing.T) {
directory := t.TempDir()
operatorPath := filepath.Join(directory, "operator.yaml")
importedPath := filepath.Join(directory, "inherited.yaml")
messagePath := filepath.Join(directory, "message.tmpl")
writeFile := func(path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
writeFile(importedPath, `profiles:
inherited:
endpoint: https://example.invalid/v1
model: model-before
temperature: 0.7
`)
writeFile(operatorPath, strings.TrimSpace(`configkit:
imports:
- path: inherited.yaml
profiles:
captured:
extends: inherited
prompts:
capture.test:
v1:
messages:
- role: user
content_file: message.tmpl
output:
format: json
validation_mode: json_schema
schema_path: schema.json
repair_attempts: 0
`))
writeFile(messagePath, "message before")
newClient := func() (*PromptKitClient, *fakePromptKitLLM) {
t.Helper()
fake := &fakePromptKitLLM{content: `{"ok":true}`}
assets := newTestPromptKitAssets(t)
if err := assets.RegisterSchemaFS(fstest.MapFS{
"schema.json": {Data: []byte(`{"type":"object"}`)},
}, "."); err != nil {
t.Fatalf("RegisterSchemaFS() error = %v", err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: assets,
ProfileFile: operatorPath,
EngineOptions: []promptkit.Option{
promptkit.WithLLMClient(fake),
},
})
if err != nil {
var loadErr *configkit.LoadError
if errors.As(err, &loadErr) {
t.Fatalf("NewPromptKitClient() error = %v (diagnostics %#v)", err, loadErr.Diagnostics())
}
t.Fatalf("NewPromptKitClient() error = %v (cause %v)", err, errors.Unwrap(err))
}
return client, fake
}
fingerprint := func(client *PromptKitClient) CheckpointFingerprint {
t.Helper()
values, err := client.LLMCheckpointFingerprints()
if err != nil || len(values) != 1 {
t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", values, err)
}
return values[0]
}
complete := func(client *PromptKitClient, fake *fakePromptKitLLM) (string, string) {
t.Helper()
var output map[string]any
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "capture.test", ProfileID: "captured", SessionID: "captured-snapshot",
}, &output)
if err != nil {
t.Fatalf("CompleteStructured() error = %v", err)
}
request := fake.lastRequest()
if len(request.Prompt.Messages) != 1 {
t.Fatalf("rendered messages = %#v", request.Prompt.Messages)
}
return request.Target.Model, request.Prompt.Messages[0].Content
}
first, firstFake := newClient()
firstFingerprint := fingerprint(first)
if model, body := complete(first, firstFake); model != "model-before" || body != "message before" {
t.Fatalf("initial execution model=%q body=%q", model, body)
}
writeFile(importedPath, `profiles:
inherited:
endpoint: https://example.invalid/v1
model: model-after
temperature: 0.7
`)
writeFile(messagePath, "message after")
if got := fingerprint(first); got != firstFingerprint {
t.Fatalf("existing engine fingerprint after source edits = %#v, want %#v", got, firstFingerprint)
}
if model, body := complete(first, firstFake); model != "model-before" || body != "message before" {
t.Fatalf("existing engine execution changed after source edits: model=%q body=%q", model, body)
}
second, secondFake := newClient()
if got := fingerprint(second); got == firstFingerprint {
t.Fatalf("new engine fingerprint = %#v, want changed imported definition/message identity", got)
}
if model, body := complete(second, secondFake); model != "model-after" || body != "message after" {
t.Fatalf("new engine execution model=%q body=%q, want edited snapshot", model, body)
}
}
func TestPromptKitClientFingerprintPreservesNullVersusZero(t *testing.T) {
fingerprintFor := func(temperature string) CheckpointFingerprint {
t.Helper()
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
content := `profiles:
parent:
endpoint: https://example.invalid/v1
model: presence-model
temperature: 0.7
selected:
extends: parent
temperature: ` + temperature + "\n"
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
ProfileFile: profilePath,
})
if err != nil {
t.Fatalf("NewPromptKitClient() error = %v", err)
}
fingerprints, err := client.LLMCheckpointFingerprints()
if err != nil || len(fingerprints) != 1 {
t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", fingerprints, err)
}
return fingerprints[0]
}
if zero, explicitNull := fingerprintFor("0"), fingerprintFor("null"); zero == explicitNull {
t.Fatalf("null and zero profile values share checkpoint identity: %#v", zero)
}
}
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint {
fingerprintFor := func(message, endpoint, parameterValue string, keyRequired bool) CheckpointFingerprint {
t.Helper()
assets := NewAssetRegistry()
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
@@ -708,7 +854,7 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi
}, "."); err != nil {
t.Fatal(err)
}
endpoint, model := "https://provider.example.test/v1", "fingerprint-model"
model := "fingerprint-model"
required := keyRequired
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: assets,
@@ -736,18 +882,23 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi
return fingerprints[0]
}
baseline := fingerprintFor("original message", "provider-secret-one", false)
if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed {
baseline := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", false)
if changed := fingerprintFor("edited message", "https://provider.example.test/v1", "provider-secret-one", false); baseline == changed {
t.Fatal("captured prompt message change did not alter configuration fingerprint")
}
if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed {
if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-two", false); baseline == changed {
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
}
if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed {
if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", true); baseline == changed {
t.Fatal("credential-policy change did not alter configuration fingerprint")
}
if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") {
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
if changed := fingerprintFor("original message", "https://other.example.test/v1", "provider-secret-one", false); baseline == changed {
t.Fatal("profile endpoint change did not alter configuration fingerprint")
}
for _, rawValue := range []string{"provider-secret-one", "original message", "provider.example.test"} {
if strings.Contains(baseline.Value, rawValue) {
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
}
}
}