Verify prepared message content hash semantics

This commit is contained in:
2026-11-20 04:18:33 +00:00
parent dd40171c01
commit 718cd05043
2 changed files with 137 additions and 0 deletions

View File

@@ -150,6 +150,8 @@ and existing handle ownership remains intact.
## Stage 3 — Verify Public Content Semantics And Exclusions ## Stage 3 — Verify Public Content Semantics And Exclusions
Status: complete.
Add focused `promptkit_test` coverage, preferably in Add focused `promptkit_test` coverage, preferably in
`message_content_hash_contract_test.go`, using real engines with small synthetic `message_content_hash_contract_test.go`, using real engines with small synthetic
definitions and an injected model client only where history or execution is definitions and an injected model client only where history or execution is

View File

@@ -0,0 +1,135 @@
package promptkit_test
import (
"context"
"testing"
"testing/fstest"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func contentHashEngine(t *testing.T, promptID, profileID, backendID, content string, output map[string]any, cache bool, opts ...promptkit.Option) *promptkit.Engine {
t.Helper()
message := map[string]any{"role": "user", "content": content}
if cache {
message["cache_control"] = map[string]any{"type": "ephemeral"}
}
config := promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("hash test", map[string]any{
"backends": []any{map[string]any{"id": backendID, "endpoint": "https://example.test/v1"}},
"profiles": []any{map[string]any{"id": profileID, "backend": backendID, "model": "model-" + profileID}},
"prompts": []any{map[string]any{"id": promptID, "default_profile": profileID, "inputs": []any{map[string]any{"name": "input", "required": false}}, "messages": []any{message}, "output": output}},
})}}}
engine, err := promptkit.NewEngine(config, opts...)
if err != nil {
t.Fatal(err)
}
return engine
}
func prepareContentHash(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest) *promptkit.PreparedRun {
t.Helper()
prepared, err := engine.Prepare(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if prepared.MessageContentHash == "" {
t.Fatal("missing content hash")
}
return prepared
}
func TestMessageContentHashIgnoresPreparationMetadata(t *testing.T) {
textOutput := map[string]any{"format": "text", "validation_mode": "none"}
baseEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, false)
base := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first"})
repeated := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first"})
other := contentHashEngine(t, "second", "beta", "two", "same", textOutput, false)
otherPrepared := prepareContentHash(t, other, promptkit.RunRequest{PromptID: "second", APIKey: "synthetic-key", Execution: &promptkit.ExecutionTargetOverride{Model: "other-model"}})
session := prepareContentHash(t, baseEngine, promptkit.RunRequest{PromptID: "first", SessionID: "other-session"})
cacheEngine := contentHashEngine(t, "first", "alpha", "one", "same", textOutput, true)
cached := prepareContentHash(t, cacheEngine, promptkit.RunRequest{PromptID: "first"})
schemas := fstest.MapFS{"schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}}
schemaEngine := contentHashEngine(t, "first", "alpha", "one", "same", map[string]any{"format": "json", "validation_mode": "json_schema", "schema_path": "schema.json"}, false, promptkit.WithSchemaFS(schemas, "."))
schema := prepareContentHash(t, schemaEngine, promptkit.RunRequest{PromptID: "first"})
for name, prepared := range map[string]*promptkit.PreparedRun{"repeat": repeated, "identifiers and model": otherPrepared, "session": session, "cache": cached, "schema": schema} {
if prepared.MessageContentHash != base.MessageContentHash {
t.Errorf("%s changed content hash", name)
}
}
if session.RenderedPromptHash == base.RenderedPromptHash || cached.RenderedPromptHash == base.RenderedPromptHash {
t.Fatal("session and cache should each change rendered prompt hash")
}
}
func TestMessageContentHashUsesRenderedInputBytes(t *testing.T) {
reader := &mutablePreparedArtifactReader{body: "body", hash: "first-hash"}
engine := contentHashEngine(t, "prompt", "profile", "backend", `{{input "input"}} {{.label}}`, map[string]any{"format": "text", "validation_mode": "none"}, false, promptkit.WithArtifactReader(reader))
request := promptkit.RunRequest{PromptID: "prompt", Inputs: map[string]promptkit.ArtifactRef{"input": promptkit.Inline("first")}, Vars: map[string]string{"label": "label", "unused": "first"}}
base := prepareContentHash(t, engine, request)
reader.set("body", "other-hash")
request.Inputs["input"] = promptkit.Inline("other")
request.Vars["unused"] = "other"
unchanged := prepareContentHash(t, engine, request)
if unchanged.MessageContentHash != base.MessageContentHash {
t.Fatal("unused variable or artifact metadata changed hash")
}
request.Vars["label"] = "changed"
if prepareContentHash(t, engine, request).MessageContentHash == base.MessageContentHash {
t.Fatal("rendered variable did not change hash")
}
request.Vars["label"] = "label"
reader.set("changed body", "other-hash")
if prepareContentHash(t, engine, request).MessageContentHash == base.MessageContentHash {
t.Fatal("rendered input did not change hash")
}
}
func TestMessageContentHashIncludesHistoryAndAppendedMessages(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "answer"}}
engine := newSimpleEngine(t, "question", promptkit.WithLLMClient(client))
initial, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
if err != nil {
t.Fatal(err)
}
request := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "extra"}}}
continued := prepareContentHash(t, engine, request)
explicit := prepareContentHash(t, engine, promptkit.RunRequest{PromptID: "prompt", AppendedMessages: []promptkit.RenderedMessage{{Role: "assistant", Content: "answer"}, {Role: "user", Content: "question"}, {Role: "user", Content: "extra"}}})
if continued.MessageContentHash != explicit.MessageContentHash {
t.Fatal("equivalent effective messages hashed differently")
}
for _, msg := range []promptkit.RenderedMessage{{Role: "assistant", Content: "extra"}, {Role: "user", Content: "changed"}} {
request.AppendedMessages = []promptkit.RenderedMessage{msg}
if prepareContentHash(t, engine, request).MessageContentHash == continued.MessageContentHash {
t.Fatal("appended message change did not affect hash")
}
}
otherClient := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "different answer"}}
otherEngine := newSimpleEngine(t, "question", promptkit.WithLLMClient(otherClient))
other, err := otherEngine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
if err != nil {
t.Fatal(err)
}
request.ContinueFrom = other.Exchange
request.AppendedMessages = []promptkit.RenderedMessage{{Role: "user", Content: "extra"}}
if prepareContentHash(t, engine, request).MessageContentHash == continued.MessageContentHash {
t.Fatal("history text change did not affect hash")
}
firstHistory, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, AppendedMessages: []promptkit.RenderedMessage{{Role: "user", Content: "history"}}})
if err != nil {
t.Fatal(err)
}
secondHistory, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true, AppendedMessages: []promptkit.RenderedMessage{{Role: "assistant", Content: "history"}}})
if err != nil {
t.Fatal(err)
}
request.ContinueFrom = firstHistory.Exchange
firstHash := prepareContentHash(t, engine, request).MessageContentHash
request.ContinueFrom = secondHistory.Exchange
if prepareContentHash(t, engine, request).MessageContentHash == firstHash {
t.Fatal("history role change did not affect hash")
}
if len(client.requests) != 3 || len(otherClient.requests) != 1 {
t.Fatal("preparation invoked the model")
}
}