Protect captured PromptKit checkpoint identity
This commit is contained in:
@@ -435,6 +435,17 @@ Prompt size: one private snapshot/digest mechanism; broader reuse tests follow.
|
|||||||
|
|
||||||
## Stage 8 — Protect Captured Identity And Resume Behavior
|
## Stage 8 — Protect Captured Identity And Resume Behavior
|
||||||
|
|
||||||
|
> Completion: complete. Added edit-after-construction coverage for an imported
|
||||||
|
> inherited profile and its source-relative prompt message: the existing client
|
||||||
|
> retains its digest and execution behavior, while a new client captures both
|
||||||
|
> edits. Fingerprint tests cover model, endpoint, credential policy, redacted
|
||||||
|
> provider parameters, prompt content, null-versus-zero presence, and local
|
||||||
|
> scheduling exclusions without exposing their values. The recorded-checkpoint
|
||||||
|
> test confirms same-snapshot reuse and stale execution misses under the new
|
||||||
|
> `promptkit_configuration` identity; existing override and chunk-plan checks
|
||||||
|
> remain passing. `GOWORK=off go test ./internal/framework/llm ./internal/cli
|
||||||
|
> ./internal/modules/integration` and `git diff --check` pass.
|
||||||
|
|
||||||
Goal: prove identity correctness across edits and checkpoint reuse.
|
Goal: prove identity correctness across edits and checkpoint reuse.
|
||||||
|
|
||||||
Inspect adapter identity tests, CLI fingerprint handoff, and existing recorded-
|
Inspect adapter identity tests, CLI fingerprint handoff, and existing recorded-
|
||||||
|
|||||||
@@ -501,11 +501,11 @@ func TestProductionLLMClientFactoryBuildsOfflineRuntime(t *testing.T) {
|
|||||||
}
|
}
|
||||||
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
|
fingerprintProvider, ok := client.(llm.CheckpointFingerprintProvider)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("production LLM client %T does not provide one profile-source checkpoint fingerprint", client)
|
t.Fatalf("production LLM client %T does not provide one captured-configuration checkpoint fingerprint", client)
|
||||||
}
|
}
|
||||||
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
|
fingerprints, err := fingerprintProvider.LLMCheckpointFingerprints()
|
||||||
if err != nil || len(fingerprints) != 1 {
|
if err != nil || len(fingerprints) != 1 {
|
||||||
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one profile-source identity", fingerprints, err)
|
t.Fatalf("production LLM checkpoint fingerprints = %#v, error = %v, want one captured-configuration identity", fingerprints, err)
|
||||||
}
|
}
|
||||||
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
|
if _, ok := client.(contracts.LLMProfileManifestProvider); !ok {
|
||||||
t.Fatalf("production LLM client %T does not provide profile manifests", client)
|
t.Fatalf("production LLM client %T does not provide profile manifests", client)
|
||||||
@@ -547,7 +547,7 @@ func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *test
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
|
if !reflect.DeepEqual(runtimeFingerprints, inspectionFingerprints) {
|
||||||
t.Fatalf("runtime profile fingerprints = %#v, inspection profile fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
t.Fatalf("runtime configuration fingerprints = %#v, inspection configuration fingerprints = %#v", runtimeFingerprints, inspectionFingerprints)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -214,7 +214,7 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
fingerprints := prepared.CheckpointFingerprints()
|
fingerprints := prepared.CheckpointFingerprints()
|
||||||
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-one"}}
|
llmFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-one"}}
|
||||||
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
settings := config.CheckpointCacheConfig{Enabled: true, Directory: t.TempDir()}
|
||||||
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
recorder, _, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, llmFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -274,16 +274,16 @@ func TestChangedSemanticSpellCatalogFingerprintCannotResumeRecordedCheckpoint(t
|
|||||||
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
t.Fatalf("changed mapping policy decision = %#v, want cold miss", decision)
|
||||||
}
|
}
|
||||||
|
|
||||||
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_profile_source", Value: "sha256:profile-source-two"}}
|
changedLLMFingerprints := []checkpoint.Fingerprint{{Name: "promptkit_configuration", Value: "sha256:configuration-two"}}
|
||||||
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
_, profileLoader, err := checkpointHandlersForRun(settings, Options{}, materialized, fingerprints, changedLLMFingerprints, []byte("same input"), nil, nil, "", "", LLMRuntimeOverrides{}, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
if _, decision := profileLoader.Source(materialized.Input.Module); decision.Reused {
|
||||||
t.Fatalf("changed PromptKit profile source decision = %#v, want cold miss", decision)
|
t.Fatalf("changed PromptKit configuration decision = %#v, want cold miss", decision)
|
||||||
}
|
}
|
||||||
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
if _, decision := profileLoader.Normalize("spells", spellnormalize.Key, normalizeDependencies); decision.Reused {
|
||||||
t.Fatalf("changed PromptKit profile normalize decision = %#v, want cold miss", decision)
|
t.Fatalf("changed PromptKit configuration normalize decision = %#v, want cold miss", decision)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -454,6 +454,7 @@ func TestPromptKitClientCheckpointFingerprintTracksCapturedConfiguration(t *test
|
|||||||
}
|
}
|
||||||
if strings.Contains(first.Value, profilePath) ||
|
if strings.Contains(first.Value, profilePath) ||
|
||||||
strings.Contains(first.Value, "model-one") ||
|
strings.Contains(first.Value, "model-one") ||
|
||||||
|
strings.Contains(first.Value, "promptkit.test") ||
|
||||||
strings.Contains(first.Value, credentialEnvironment) {
|
strings.Contains(first.Value, credentialEnvironment) {
|
||||||
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
|
t.Fatalf("configuration fingerprint exposes source details: %#v", first)
|
||||||
}
|
}
|
||||||
@@ -693,8 +694,153 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientFingerprintAndExecutionRetainImportedSnapshot(t *testing.T) {
|
||||||
|
directory := t.TempDir()
|
||||||
|
operatorPath := filepath.Join(directory, "operator.yaml")
|
||||||
|
importedPath := filepath.Join(directory, "inherited.yaml")
|
||||||
|
messagePath := filepath.Join(directory, "message.tmpl")
|
||||||
|
writeFile := func(path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeFile(importedPath, `profiles:
|
||||||
|
inherited:
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: model-before
|
||||||
|
temperature: 0.7
|
||||||
|
`)
|
||||||
|
writeFile(operatorPath, strings.TrimSpace(`configkit:
|
||||||
|
imports:
|
||||||
|
- path: inherited.yaml
|
||||||
|
profiles:
|
||||||
|
captured:
|
||||||
|
extends: inherited
|
||||||
|
prompts:
|
||||||
|
capture.test:
|
||||||
|
v1:
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content_file: message.tmpl
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: schema.json
|
||||||
|
repair_attempts: 0
|
||||||
|
`))
|
||||||
|
writeFile(messagePath, "message before")
|
||||||
|
newClient := func() (*PromptKitClient, *fakePromptKitLLM) {
|
||||||
|
t.Helper()
|
||||||
|
fake := &fakePromptKitLLM{content: `{"ok":true}`}
|
||||||
|
assets := newTestPromptKitAssets(t)
|
||||||
|
if err := assets.RegisterSchemaFS(fstest.MapFS{
|
||||||
|
"schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
|
}, "."); err != nil {
|
||||||
|
t.Fatalf("RegisterSchemaFS() error = %v", err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
|
Assets: assets,
|
||||||
|
ProfileFile: operatorPath,
|
||||||
|
EngineOptions: []promptkit.Option{
|
||||||
|
promptkit.WithLLMClient(fake),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
var loadErr *configkit.LoadError
|
||||||
|
if errors.As(err, &loadErr) {
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v (diagnostics %#v)", err, loadErr.Diagnostics())
|
||||||
|
}
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v (cause %v)", err, errors.Unwrap(err))
|
||||||
|
}
|
||||||
|
return client, fake
|
||||||
|
}
|
||||||
|
fingerprint := func(client *PromptKitClient) CheckpointFingerprint {
|
||||||
|
t.Helper()
|
||||||
|
values, err := client.LLMCheckpointFingerprints()
|
||||||
|
if err != nil || len(values) != 1 {
|
||||||
|
t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", values, err)
|
||||||
|
}
|
||||||
|
return values[0]
|
||||||
|
}
|
||||||
|
complete := func(client *PromptKitClient, fake *fakePromptKitLLM) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
var output map[string]any
|
||||||
|
_, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||||
|
PromptID: "capture.test", ProfileID: "captured", SessionID: "captured-snapshot",
|
||||||
|
}, &output)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompleteStructured() error = %v", err)
|
||||||
|
}
|
||||||
|
request := fake.lastRequest()
|
||||||
|
if len(request.Prompt.Messages) != 1 {
|
||||||
|
t.Fatalf("rendered messages = %#v", request.Prompt.Messages)
|
||||||
|
}
|
||||||
|
return request.Target.Model, request.Prompt.Messages[0].Content
|
||||||
|
}
|
||||||
|
|
||||||
|
first, firstFake := newClient()
|
||||||
|
firstFingerprint := fingerprint(first)
|
||||||
|
if model, body := complete(first, firstFake); model != "model-before" || body != "message before" {
|
||||||
|
t.Fatalf("initial execution model=%q body=%q", model, body)
|
||||||
|
}
|
||||||
|
writeFile(importedPath, `profiles:
|
||||||
|
inherited:
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: model-after
|
||||||
|
temperature: 0.7
|
||||||
|
`)
|
||||||
|
writeFile(messagePath, "message after")
|
||||||
|
if got := fingerprint(first); got != firstFingerprint {
|
||||||
|
t.Fatalf("existing engine fingerprint after source edits = %#v, want %#v", got, firstFingerprint)
|
||||||
|
}
|
||||||
|
if model, body := complete(first, firstFake); model != "model-before" || body != "message before" {
|
||||||
|
t.Fatalf("existing engine execution changed after source edits: model=%q body=%q", model, body)
|
||||||
|
}
|
||||||
|
second, secondFake := newClient()
|
||||||
|
if got := fingerprint(second); got == firstFingerprint {
|
||||||
|
t.Fatalf("new engine fingerprint = %#v, want changed imported definition/message identity", got)
|
||||||
|
}
|
||||||
|
if model, body := complete(second, secondFake); model != "model-after" || body != "message after" {
|
||||||
|
t.Fatalf("new engine execution model=%q body=%q, want edited snapshot", model, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptKitClientFingerprintPreservesNullVersusZero(t *testing.T) {
|
||||||
|
fingerprintFor := func(temperature string) CheckpointFingerprint {
|
||||||
|
t.Helper()
|
||||||
|
profilePath := filepath.Join(t.TempDir(), "profiles.yaml")
|
||||||
|
content := `profiles:
|
||||||
|
parent:
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: presence-model
|
||||||
|
temperature: 0.7
|
||||||
|
selected:
|
||||||
|
extends: parent
|
||||||
|
temperature: ` + temperature + "\n"
|
||||||
|
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
|
Assets: newTestPromptKitAssets(t),
|
||||||
|
ProfileFile: profilePath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewPromptKitClient() error = %v", err)
|
||||||
|
}
|
||||||
|
fingerprints, err := client.LLMCheckpointFingerprints()
|
||||||
|
if err != nil || len(fingerprints) != 1 {
|
||||||
|
t.Fatalf("LLMCheckpointFingerprints() = (%#v, %v)", fingerprints, err)
|
||||||
|
}
|
||||||
|
return fingerprints[0]
|
||||||
|
}
|
||||||
|
if zero, explicitNull := fingerprintFor("0"), fingerprintFor("null"); zero == explicitNull {
|
||||||
|
t.Fatalf("null and zero profile values share checkpoint identity: %#v", zero)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
|
func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfile(t *testing.T) {
|
||||||
fingerprintFor := func(message, parameterValue string, keyRequired bool) CheckpointFingerprint {
|
fingerprintFor := func(message, endpoint, parameterValue string, keyRequired bool) CheckpointFingerprint {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
assets := NewAssetRegistry()
|
assets := NewAssetRegistry()
|
||||||
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
|
promptDocument := "prompts:\n fingerprint.test:\n v1:\n messages:\n - role: user\n content: " + fmt.Sprintf("%q", message) + "\n output:\n format: json\n validation_mode: json_schema\n schema_path: result.json\n"
|
||||||
@@ -708,7 +854,7 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi
|
|||||||
}, "."); err != nil {
|
}, "."); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
endpoint, model := "https://provider.example.test/v1", "fingerprint-model"
|
model := "fingerprint-model"
|
||||||
required := keyRequired
|
required := keyRequired
|
||||||
client, err := NewPromptKitClient(PromptKitClientConfig{
|
client, err := NewPromptKitClient(PromptKitClientConfig{
|
||||||
Assets: assets,
|
Assets: assets,
|
||||||
@@ -736,18 +882,23 @@ func TestPromptKitConfigurationFingerprintIncludesCapturedPromptAndResolvedProfi
|
|||||||
return fingerprints[0]
|
return fingerprints[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
baseline := fingerprintFor("original message", "provider-secret-one", false)
|
baseline := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", false)
|
||||||
if changed := fingerprintFor("edited message", "provider-secret-one", false); baseline == changed {
|
if changed := fingerprintFor("edited message", "https://provider.example.test/v1", "provider-secret-one", false); baseline == changed {
|
||||||
t.Fatal("captured prompt message change did not alter configuration fingerprint")
|
t.Fatal("captured prompt message change did not alter configuration fingerprint")
|
||||||
}
|
}
|
||||||
if changed := fingerprintFor("original message", "provider-secret-two", false); baseline == changed {
|
if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-two", false); baseline == changed {
|
||||||
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
|
t.Fatal("resolved provider parameter change did not alter configuration fingerprint")
|
||||||
}
|
}
|
||||||
if changed := fingerprintFor("original message", "provider-secret-one", true); baseline == changed {
|
if changed := fingerprintFor("original message", "https://provider.example.test/v1", "provider-secret-one", true); baseline == changed {
|
||||||
t.Fatal("credential-policy change did not alter configuration fingerprint")
|
t.Fatal("credential-policy change did not alter configuration fingerprint")
|
||||||
}
|
}
|
||||||
if strings.Contains(baseline.Value, "provider-secret-one") || strings.Contains(baseline.Value, "original message") {
|
if changed := fingerprintFor("original message", "https://other.example.test/v1", "provider-secret-one", false); baseline == changed {
|
||||||
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
|
t.Fatal("profile endpoint change did not alter configuration fingerprint")
|
||||||
|
}
|
||||||
|
for _, rawValue := range []string{"provider-secret-one", "original message", "provider.example.test"} {
|
||||||
|
if strings.Contains(baseline.Value, rawValue) {
|
||||||
|
t.Fatalf("configuration fingerprint leaked captured content: %#v", baseline)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user