From 7ea9fe12b9a77939f1ee9ab969e559510b6f3d14 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 13:08:56 +0000 Subject: [PATCH] Protect captured PromptKit checkpoint identity --- docs/roadmap/implementation.md | 11 ++ internal/cli/production_contract_test.go | 6 +- .../spell_catalog_identity_contract_test.go | 8 +- .../framework/llm/promptkit_client_test.go | 167 +++++++++++++++++- 4 files changed, 177 insertions(+), 15 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index c5112320..1026fd86 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -435,6 +435,17 @@ Prompt size: one private snapshot/digest mechanism; broader reuse tests follow. ## Stage 8 — Protect Captured Identity And Resume Behavior +> Completion: complete. Added edit-after-construction coverage for an imported +> inherited profile and its source-relative prompt message: the existing client +> retains its digest and execution behavior, while a new client captures both +> edits. Fingerprint tests cover model, endpoint, credential policy, redacted +> provider parameters, prompt content, null-versus-zero presence, and local +> scheduling exclusions without exposing their values. The recorded-checkpoint +> test confirms same-snapshot reuse and stale execution misses under the new +> `promptkit_configuration` identity; existing override and chunk-plan checks +> remain passing. `GOWORK=off go test ./internal/framework/llm ./internal/cli +> ./internal/modules/integration` and `git diff --check` pass. + Goal: prove identity correctness across edits and checkpoint reuse. Inspect adapter identity tests, CLI fingerprint handoff, and existing recorded- diff --git a/internal/cli/production_contract_test.go b/internal/cli/production_contract_test.go index 6cf04875..4e27730a 100644 --- a/internal/cli/production_contract_test.go +++ b/internal/cli/production_contract_test.go @@ -501,11 +501,11 @@ func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) { } fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider) if !ok { - t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client) + t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client) } fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints() if err != nil || len(fingerprints) != 1 { - t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err) + t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err) } if _, ok := client.(contracts.LLMProfileManifestProvider); !ok { t.Fatalf("production LLM client %T does not provide profile manifests", client) @@ -547,7 +547,7 @@ func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *test t.Fatal(err) } if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) { - t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints) + t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints) } } diff --git a/internal/cli/spell_catalog_identity_contract_test.go b/internal/cli/spell_catalog_identity_contract_test.go index 0fe49a2e..8309eb69 100644 --- a/internal/cli/spell_catalog_identity_contract_test.go +++ b/internal/cli/spell_catalog_identity_contract_test.go @@ -214,7 +214,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t t.Fatal(err) } fingerprints := prepared.CheckpointFingerprints() - llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}} + llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}} settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()} recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false) if err != nil { @@ -274,16 +274,16 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision) } - changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}} + changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}} _, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true) if err != nil { t.Fatal(err) } if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused { - t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision) + t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision) } if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused { - t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision) + t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision) } } diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index b8387b50..3916374d 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -454,6 +454,7 @@ func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *test } if strings.Contains(first.Value, profilePath) || strings.Contains(first.Value, "model-one") || + strings.Contains(first.Value, "promptkit.test") || strings.Contains(first.Value, credentialEnvironment) { t.Fatalf("configuration fingerprint exposes source details: %#v", first) } @@ -693,8 +694,153 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test } } +func TestPromptKitClientFingerprintAndExecutionRetainImportedSnapshot(t *testing.T) { + directory := t.TempDir() + operatorPath := filepath.Join(directory, "operator.yaml") + importedPath := filepath.Join(directory, "inherited.yaml") + messagePath := filepath.Join(directory, "message.tmpl") + writeFile := func(path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + writeFile(importedPath, `profiles: + inherited: + endpoint: https://example.invalid/v1 + model: model-before + temperature: 0.7 +`) + writeFile(operatorPath, strings.TrimSpace(`configkit: + imports: + - path: inherited.yaml +profiles: + captured: + extends: inherited +prompts: + capture.test: + v1: + messages: + - role: user + content_file: message.tmpl + output: + format: json + validation_mode: json_schema + schema_path: schema.json + repair_attempts: 0 + `)) + writeFile(messagePath, "message before") + newClient := func() (*PromptKitClient, *fakePromptKitLLM) { + t.Helper() + fake := &fakePromptKitLLM{content: `{"ok":true}`} + assets := newTestPromptKitAssets(t) + if err := assets.RegisterSchemaFS(fstest.MapFS{ + "schema.json": {Data: []byte(`{"type":"object"}`)}, + }, "."); err != nil { + t.Fatalf("RegisterSchemaFS() error = %v", err) + } + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: assets, + ProfileFile: operatorPath, + EngineOptions: []promptkit.Option{ + promptkit.WithLLMClient(fake), + }, + }) + if err != nil { + var loadErr *configkit.LoadError + if errors.As(err, &loadErr) { + t.Fatalf("NewPromptKitClient() error = %v (diagnostics %#v)", err, loadErr.Diagnostics()) + } + t.Fatalf("NewPromptKitClient() error = %v (cause %v)", err, errors.Unwrap(err)) + } + return client, fake + } + fingerprint := func(client *PromptKitClient) CheckpointFingerprint { + t.Helper() + values, err := client.LLMCheckpointFingerprints() + if err != nil || len(values) != 1 { + t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", values, err) + } + return values[0] + } + complete := func(client *PromptKitClient, fake *fakePromptKitLLM) (string, string) { + t.Helper() + var output map[string]any + _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "capture.test", ProfileID: "captured", SessionID: "captured-snapshot", + }, &output) + if err != nil { + t.Fatalf("CompleteStructured() error = %v", err) + } + request := fake.lastRequest() + if len(request.Prompt.Messages) != 1 { + t.Fatalf("rendered messages = %#v", request.Prompt.Messages) + } + return request.Target.Model, request.Prompt.Messages[0].Content + } + + first, firstFake := newClient() + firstFingerprint := fingerprint(first) + if model, body := complete(first, firstFake); model != "model-before" || body != "message before" { + t.Fatalf("initial execution model=%q body=%q", model, body) + } + writeFile(importedPath, `profiles: + inherited: + endpoint: https://example.invalid/v1 + model: model-after + temperature: 0.7 +`) + writeFile(messagePath, "message after") + if got := fingerprint(first); got != firstFingerprint { + t.Fatalf("existing engine fingerprint after source edits = %#v, want %#v", got, firstFingerprint) + } + if model, body := complete(first, firstFake); model != "model-before" || body != "message before" { + t.Fatalf("existing engine execution changed after source edits: model=%q body=%q", model, body) + } + second, secondFake := newClient() + if got := fingerprint(second); got == firstFingerprint { + t.Fatalf("new engine fingerprint = %#v, want changed imported definition/message identity", got) + } + if model, body := complete(second, secondFake); model != "model-after" || body != "message after" { + t.Fatalf("new engine execution model=%q body=%q, want edited snapshot", model, body) + } +} + +func TestPromptKitClientFingerprintPreservesNullVersusZero(t *testing.T) { + fingerprintFor := func(temperature string) CheckpointFingerprint { + t.Helper() + profilePath := filepath.Join(t.TempDir(), "profiles.yaml") + content := `profiles: + parent: + endpoint: https://example.invalid/v1 + model: presence-model + temperature: 0.7 + selected: + extends: parent + temperature: ` + temperature + "\n" + if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + client, err := NewPromptKitClient(PromptKitClientConfig{ + Assets: newTestPromptKitAssets(t), + ProfileFile: profilePath, + }) + if err != nil { + t.Fatalf("NewPromptKitClient() error = %v", err) + } + fingerprints, err := client.LLMCheckpointFingerprints() + if err != nil || len(fingerprints) != 1 { + t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", fingerprints, err) + } + return fingerprints[0] + } + if zero, explicitNull := fingerprintFor("0"), fingerprintFor("null"); zero == explicitNull { + t.Fatalf("null and zero profile values share checkpoint identity: %#v", zero) + } +} + func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) { - fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint { + fingerprintFor := func(message, endpoint, parameterValue string, keyRequired bool) CheckpointFingerprint { t.Helper() assets := NewAssetRegistry() promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n" @@ -708,7 +854,7 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi }, "."); err != nil { t.Fatal(err) } - endpoint, model := "https://provider.example.test/v1", "fingerprint-model" + model := "fingerprint-model" required := keyRequired client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: assets, @@ -736,18 +882,23 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi return fingerprints[0] } - baseline := fingerprintFor("original message", "provider-secret-one", false) - if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed { + baseline := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", false) + if changed := fingerprintFor("edited message", "https://provider.example.test/v1", "provider-secret-one", false); baseline == changed { t.Fatal("captured prompt message change did not alter configuration fingerprint") } - if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed { + if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-two", false); baseline == changed { t.Fatal("resolved provider parameter change did not alter configuration fingerprint") } - if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed { + if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", true); baseline == changed { t.Fatal("credential-policy change did not alter configuration fingerprint") } - if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") { - t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline) + if changed := fingerprintFor("original message", "https://other.example.test/v1", "provider-secret-one", false); baseline == changed { + t.Fatal("profile endpoint change did not alter configuration fingerprint") + } + for _, rawValue := range []string{"provider-secret-one", "original message", "provider.example.test"} { + if strings.Contains(baseline.Value, rawValue) { + t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline) + } } }