base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Move renderer tests to captured templates

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-05 19:59:34 +00:00
parent acaf2ccd8b
commit d78474a3bf
5 changed files with 125 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/configuration/targets_test.go
View File

@@ -75,7 +75,7 @@ func TestRequestCompletesLoadedPartialProfile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
runner := usecase.NewRunner(state, state, nil, prompt.NewGoRenderer(), nil, nil, nil)
runner := usecase.NewRunner(state, state, nil, prompt.NewCompiledRenderer(state), nil, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "example", ProfileID: "partial",
Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"},

122
internal/prompt/renderer_test.go
View File

@@ -14,7 +14,6 @@ import (
)
func TestGoRenderer_Render(t *testing.T) {
renderer := NewGoRenderer()
ctx := context.Background()
inputs := map[string]*domain.Artifact{
@@ -33,7 +32,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -53,7 +52,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -71,7 +70,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -99,7 +98,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -129,7 +128,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -151,7 +150,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -169,7 +168,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, map[string]string{
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": "agent-session-123",
})
@@ -190,7 +189,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -208,7 +207,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
@@ -223,7 +222,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
_, err := renderer.Render(ctx, def, inputs, map[string]string{
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": strings.Repeat("x", domain.SessionIDMaxLength+1),
})
@@ -241,7 +240,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
_, err := renderer.Render(ctx, def, inputs, map[string]string{
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{
"tone": "concise",
"session_id": "session" + string([]byte{0xff}),
})
@@ -258,7 +257,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -278,7 +277,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
res, err := renderer.Render(ctx, def, inputs, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -298,7 +297,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
@@ -315,7 +314,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
_, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars)
_, err := capturedTestRenderer(t, def).Render(ctx, def, map[string]*domain.Artifact{}, vars)
if !errors.Is(err, ErrMissingRequiredInput) {
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
}
@@ -329,9 +328,9 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrInvalidTemplate) {
t.Fatalf("expected ErrInvalidTemplate, got %v", err)
_, err := CompileTemplate("message", def.Templates[0].Content)
if err == nil {
t.Fatal("expected template compilation to fail")
}
})
@@ -343,7 +342,7 @@ func TestGoRenderer_Render(t *testing.T) {
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrRenderFailure) {
t.Fatalf("expected ErrRenderFailure, got %v", err)
}
@@ -359,7 +358,7 @@ func TestGoRenderer_Render(t *testing.T) {
{Role: "", Content: "Hello"},
},
}
_, err := renderer.Render(ctx, def, inputs, vars)
_, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars)
if !errors.Is(err, ErrInvalidMessageRole) {
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
}
@@ -371,7 +370,7 @@ func TestGoRenderer_Render(t *testing.T) {
{Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"},
},
}
res, err := renderer.Render(ctx, def, nil, vars)
res, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -385,7 +384,7 @@ func TestGoRenderer_Render(t *testing.T) {
def := &domain.PromptDefinition{
Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}},
}
_, err := renderer.Render(ctx, def, nil, vars)
_, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars)
if !errors.Is(err, ErrInvalidMessageRole) {
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
}
@@ -396,9 +395,9 @@ func TestGoRenderer_Render(t *testing.T) {
}
func TestGoRendererCancellation(t *testing.T) {
t.Run("before session parsing", func(t *testing.T) {
t.Run("before rendering", func(t *testing.T) {
definition := &domain.PromptDefinition{
SessionID: "{{ malformed",
SessionID: "{{.session}}",
Templates: []domain.PromptMessageTemplate{
{Role: "user", Content: "not rendered"},
},
@@ -406,18 +405,11 @@ func TestGoRendererCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
renderer := capturedTestRenderer(t, definition)
result, err := renderer.Render(ctx, definition, nil, nil)
if result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
}
if errors.Is(err, ErrInvalidTemplate) {
t.Fatalf("pre-canceled render parsed the malformed session: %v", err)
}
result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil)
if result != nil || !errors.Is(err, ErrInvalidTemplate) {
t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err)
}
})
t.Run("during artifact text conversion", func(t *testing.T) {
@@ -447,14 +439,15 @@ func TestGoRendererCancellation(t *testing.T) {
},
}
counter := &checkCountingContext{Context: context.Background()}
if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil {
renderer := capturedTestRenderer(t, definition)
if _, err := renderer.Render(counter, definition, nil, nil); err != nil {
t.Fatalf("count render checkpoints: %v", err)
}
// The final three checks occur after template execution, after the
// message is assembled, and immediately before publication.
ctx := newCancelOnCheckContext(counter.checks - 2)
result, err := NewGoRenderer().Render(ctx, definition, nil, nil)
result, err := renderer.Render(ctx, definition, nil, nil)
if result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err)
}
@@ -472,7 +465,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
},
}
first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
renderer := capturedTestRenderer(t, definition)
first, err := renderer.Render(context.Background(), definition, inputs, nil)
if err != nil {
t.Fatalf("first render: %v", err)
}
@@ -488,7 +482,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) {
t.Fatal("completed render aliases the artifact body")
}
second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil)
second, err := renderer.Render(context.Background(), definition, inputs, nil)
if err != nil {
t.Fatalf("second render: %v", err)
}
@@ -498,7 +492,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
}
nilInputs := map[string]*domain.Artifact{"document": nil}
result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil)
result, err := renderer.Render(context.Background(), definition, nilInputs, nil)
if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) {
t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err)
}
@@ -559,11 +553,56 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
}
}
func TestCompiledRendererRequiresCapturedTemplates(t *testing.T) {
message, err := CompileTemplate("message", "hello")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
lookup TemplateLookup
sessionID string
}{
{name: "missing lookup"},
{name: "missing prompt", lookup: compiledRendererTestLookup{id: "other"}},
{name: "missing message", lookup: compiledRendererTestLookup{id: "prompt"}},
{name: "nil message", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{nil}}}},
{name: "missing session", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{message}}}, sessionID: "session"},
} {
t.Run(tc.name, func(t *testing.T) {
definition := &domain.PromptDefinition{ID: "prompt", SessionID: tc.sessionID, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hello"}}}
if _, err := NewCompiledRenderer(tc.lookup).Render(context.Background(), definition, nil, nil); !errors.Is(err, ErrRenderFailure) {
t.Fatalf("render error = %v, want render failure", err)
}
})
}
}
type compiledRendererTestLookup struct {
id string
templates CompiledTemplates
}
func capturedTestRenderer(t testing.TB, definition *domain.PromptDefinition) Renderer {
t.Helper()
var compiled CompiledTemplates
var err error
if definition.SessionID != "" {
compiled.Session, err = CompileTemplate("session", definition.SessionID)
if err != nil {
t.Fatal(err)
}
}
for _, message := range definition.Templates {
parsed, err := CompileTemplate("message", message.Content)
if err != nil {
t.Fatal(err)
}
compiled.Messages = append(compiled.Messages, parsed)
}
return NewCompiledRenderer(compiledRendererTestLookup{id: definition.ID, templates: compiled})
}
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) {
if id != lookup.id {
return CompiledTemplates{}, false
@@ -571,8 +610,7 @@ func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (Compil
return lookup.templates, true
}
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
// template-parsing fallback rather than the captured compiled path.
// BenchmarkGoRendererArtifactReferences measures rendering captured templates.
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
inputs := map[string]*domain.Artifact{
@@ -604,7 +642,7 @@ func BenchmarkGoRendererArtifactReferences(b *testing.B) {
for _, tc := range tests {
b.Run(tc.name, func(b *testing.B) {
renderer := NewGoRenderer()
renderer := capturedTestRenderer(b, tc.definition)
b.ReportAllocs()
b.SetBytes(int64(len(body)))
for range b.N {

4
internal/prompt/requirements_test.go
View File

@@ -314,7 +314,7 @@ func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
"active": {Name: "active", Body: []byte("active")},
"inactive": {Name: "inactive", Body: []byte("inactive")},
}
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
if err != nil {
t.Fatal(err)
}
@@ -526,7 +526,7 @@ func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
}
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars)
if err != nil {
t.Fatal(err)
}

41
internal/usecase/runner_test.go
View File

@@ -13,6 +13,7 @@ import (
"strings"
"sync"
"testing"
"text/template"
"time"
"go.maximumdirect.net/promptkit/internal/capacity"
@@ -41,6 +42,36 @@ type staticPromptRepo struct {
def *domain.PromptDefinition
}
type capturedTemplateFixture struct {
id string
templates prompt.CompiledTemplates
}
func (fixture capturedTemplateFixture) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) {
return fixture.templates, id == fixture.id
}
func testCompiledRenderer(t *testing.T, definition *domain.PromptDefinition) prompt.Renderer {
t.Helper()
var templates prompt.CompiledTemplates
if definition.SessionID != "" {
var err error
templates.Session, err = prompt.CompileTemplate("session", definition.SessionID)
if err != nil {
t.Fatal(err)
}
}
templates.Messages = make([]*template.Template, 0, len(definition.Templates))
for _, message := range definition.Templates {
compiled, err := prompt.CompileTemplate("message", message.Content)
if err != nil {
t.Fatal(err)
}
templates.Messages = append(templates.Messages, compiled)
}
return prompt.NewCompiledRenderer(capturedTemplateFixture{id: definition.ID, templates: templates})
}
func (r staticPromptRepo) GetPromptSnapshot(
context.Context,
string,
@@ -449,7 +480,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
testCompiledRenderer(t, def),
&fakeLLM{forbid: true},
nil, nil)
@@ -496,7 +527,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
testCompiledRenderer(t, def),
&fakeLLM{forbid: true},
nil, nil)
@@ -526,7 +557,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
testCompiledRenderer(t, def),
&fakeLLM{forbid: true},
nil, nil)
@@ -886,7 +917,7 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
testCompiledRenderer(t, def),
&fakeLLM{forbid: true},
nil, nil)
@@ -913,7 +944,7 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
testCompiledRenderer(t, def),
&fakeLLM{forbid: true},
nil, nil)