From d78474a3bf2d81a7dae82f107b50c15f5ab9ec1d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 5 Nov 2026 19:59:34 +0000 Subject: [PATCH] Move renderer tests to captured templates --- docs/roadmap/implementation.md | 6 ++ internal/configuration/targets_test.go | 2 +- internal/prompt/renderer_test.go | 122 ++++++++++++++++--------- internal/prompt/requirements_test.go | 4 +- internal/usecase/runner_test.go | 41 ++++++++- 5 files changed, 125 insertions(+), 50 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 0046504..e04f4a5 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -185,6 +185,12 @@ are deliberately migrated in Stage 6 rather than accommodated in production. ## Stage 6: Move Renderer Tests Onto Captured Templates +**Completed:** Renderer fixtures and benchmarks now compile captured templates +before rendering; invalid syntax, missing captures, cancellation, session, and +artifact behavior remain covered. Go 1.27.3 `go test ./internal/prompt +./internal/configuration ./internal/usecase` passed, resolving the deferred +repository and renderer checks. + **Finding:** 3. **Scope:** prompt rendering tests/benchmarks and their use-case/configuration callers. 1. Use `CompileTemplate` and a small `TemplateLookup` fixture, or a loaded diff --git a/internal/configuration/targets_test.go b/internal/configuration/targets_test.go index 26e9df2..fad5862 100644 --- a/internal/configuration/targets_test.go +++ b/internal/configuration/targets_test.go @@ -75,7 +75,7 @@ func TestRequestCompletesLoadedPartialProfile(t *testing.T) { if err != nil { t.Fatal(err) } - runner := usecase.NewRunner(state, state, nil, prompt.NewGoRenderer(), nil, nil, nil) + runner := usecase.NewRunner(state, state, nil, prompt.NewCompiledRenderer(state), nil, nil, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "example", ProfileID: "partial", Execution: &domain.ExecutionTargetOverride{Endpoint: "https://example.invalid/v1", Model: "request-model"}, diff --git a/internal/prompt/renderer_test.go b/internal/prompt/renderer_test.go index e38cf1d..f6a4fd8 100644 --- a/internal/prompt/renderer_test.go +++ b/internal/prompt/renderer_test.go @@ -14,7 +14,6 @@ import ( ) func TestGoRenderer_Render(t *testing.T) { - renderer := NewGoRenderer() ctx := context.Background() inputs := map[string]*domain.Artifact{ @@ -33,7 +32,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -53,7 +52,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -71,7 +70,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -99,7 +98,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -129,7 +128,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -151,7 +150,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -169,7 +168,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, map[string]string{ + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{ "tone": "concise", "session_id": "agent-session-123", }) @@ -190,7 +189,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -208,7 +207,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - _, err := renderer.Render(ctx, def, inputs, vars) + _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if !errors.Is(err, ErrRenderFailure) { t.Fatalf("expected ErrRenderFailure, got %v", err) } @@ -223,7 +222,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - _, err := renderer.Render(ctx, def, inputs, map[string]string{ + _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{ "tone": "concise", "session_id": strings.Repeat("x", domain.SessionIDMaxLength+1), }) @@ -241,7 +240,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - _, err := renderer.Render(ctx, def, inputs, map[string]string{ + _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, map[string]string{ "tone": "concise", "session_id": "session" + string([]byte{0xff}), }) @@ -258,7 +257,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -278,7 +277,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - res, err := renderer.Render(ctx, def, inputs, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -298,7 +297,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - _, err := renderer.Render(ctx, def, inputs, vars) + _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if !errors.Is(err, ErrRenderFailure) { t.Fatalf("expected ErrRenderFailure, got %v", err) } @@ -315,7 +314,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - _, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars) + _, err := capturedTestRenderer(t, def).Render(ctx, def, map[string]*domain.Artifact{}, vars) if !errors.Is(err, ErrMissingRequiredInput) { t.Fatalf("expected ErrMissingRequiredInput, got %v", err) } @@ -329,9 +328,9 @@ func TestGoRenderer_Render(t *testing.T) { }, } - _, err := renderer.Render(ctx, def, inputs, vars) - if !errors.Is(err, ErrInvalidTemplate) { - t.Fatalf("expected ErrInvalidTemplate, got %v", err) + _, err := CompileTemplate("message", def.Templates[0].Content) + if err == nil { + t.Fatal("expected template compilation to fail") } }) @@ -343,7 +342,7 @@ func TestGoRenderer_Render(t *testing.T) { }, } - _, err := renderer.Render(ctx, def, inputs, vars) + _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if !errors.Is(err, ErrRenderFailure) { t.Fatalf("expected ErrRenderFailure, got %v", err) } @@ -359,7 +358,7 @@ func TestGoRenderer_Render(t *testing.T) { {Role: "", Content: "Hello"}, }, } - _, err := renderer.Render(ctx, def, inputs, vars) + _, err := capturedTestRenderer(t, def).Render(ctx, def, inputs, vars) if !errors.Is(err, ErrInvalidMessageRole) { t.Fatalf("expected ErrInvalidMessageRole, got %v", err) } @@ -371,7 +370,7 @@ func TestGoRenderer_Render(t *testing.T) { {Role: " \u2003DeVeLoPeR\u2003 ", Content: "Hello"}, }, } - res, err := renderer.Render(ctx, def, nil, vars) + res, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -385,7 +384,7 @@ func TestGoRenderer_Render(t *testing.T) { def := &domain.PromptDefinition{ Templates: []domain.PromptMessageTemplate{{Role: unsupportedRole, Content: "Hello"}}, } - _, err := renderer.Render(ctx, def, nil, vars) + _, err := capturedTestRenderer(t, def).Render(ctx, def, nil, vars) if !errors.Is(err, ErrInvalidMessageRole) { t.Fatalf("expected ErrInvalidMessageRole, got %v", err) } @@ -396,9 +395,9 @@ func TestGoRenderer_Render(t *testing.T) { } func TestGoRendererCancellation(t *testing.T) { - t.Run("before session parsing", func(t *testing.T) { + t.Run("before rendering", func(t *testing.T) { definition := &domain.PromptDefinition{ - SessionID: "{{ malformed", + SessionID: "{{.session}}", Templates: []domain.PromptMessageTemplate{ {Role: "user", Content: "not rendered"}, }, @@ -406,18 +405,11 @@ func TestGoRendererCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - result, err := NewGoRenderer().Render(ctx, definition, nil, nil) + renderer := capturedTestRenderer(t, definition) + result, err := renderer.Render(ctx, definition, nil, nil) if result != nil || !errors.Is(err, context.Canceled) { t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err) } - if errors.Is(err, ErrInvalidTemplate) { - t.Fatalf("pre-canceled render parsed the malformed session: %v", err) - } - - result, err = NewGoRenderer().Render(context.Background(), definition, nil, nil) - if result != nil || !errors.Is(err, ErrInvalidTemplate) { - t.Fatalf("active render result=%#v err=%v, want nil/ErrInvalidTemplate", result, err) - } }) t.Run("during artifact text conversion", func(t *testing.T) { @@ -447,14 +439,15 @@ func TestGoRendererCancellation(t *testing.T) { }, } counter := &checkCountingContext{Context: context.Background()} - if _, err := NewGoRenderer().Render(counter, definition, nil, nil); err != nil { + renderer := capturedTestRenderer(t, definition) + if _, err := renderer.Render(counter, definition, nil, nil); err != nil { t.Fatalf("count render checkpoints: %v", err) } // The final three checks occur after template execution, after the // message is assembled, and immediately before publication. ctx := newCancelOnCheckContext(counter.checks - 2) - result, err := NewGoRenderer().Render(ctx, definition, nil, nil) + result, err := renderer.Render(ctx, definition, nil, nil) if result != nil || !errors.Is(err, context.Canceled) { t.Fatalf("result=%#v err=%v, want nil/context.Canceled", result, err) } @@ -472,7 +465,8 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) { }, } - first, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil) + renderer := capturedTestRenderer(t, definition) + first, err := renderer.Render(context.Background(), definition, inputs, nil) if err != nil { t.Fatalf("first render: %v", err) } @@ -488,7 +482,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) { if bytes.Equal([]byte(first.Messages[0].Content), append(append(append([]byte(nil), body...), '|'), body...)) { t.Fatal("completed render aliases the artifact body") } - second, err := NewGoRenderer().Render(context.Background(), definition, inputs, nil) + second, err := renderer.Render(context.Background(), definition, inputs, nil) if err != nil { t.Fatalf("second render: %v", err) } @@ -498,7 +492,7 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) { } nilInputs := map[string]*domain.Artifact{"document": nil} - result, err := NewGoRenderer().Render(context.Background(), definition, nilInputs, nil) + result, err := renderer.Render(context.Background(), definition, nilInputs, nil) if result != nil || !errors.Is(err, ErrUnknownInput) || !errors.Is(err, ErrRenderFailure) { t.Fatalf("nil input result=%#v err=%v, want ErrUnknownInput and ErrRenderFailure", result, err) } @@ -559,11 +553,56 @@ func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) { } } +func TestCompiledRendererRequiresCapturedTemplates(t *testing.T) { + message, err := CompileTemplate("message", "hello") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + lookup TemplateLookup + sessionID string + }{ + {name: "missing lookup"}, + {name: "missing prompt", lookup: compiledRendererTestLookup{id: "other"}}, + {name: "missing message", lookup: compiledRendererTestLookup{id: "prompt"}}, + {name: "nil message", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{nil}}}}, + {name: "missing session", lookup: compiledRendererTestLookup{id: "prompt", templates: CompiledTemplates{Messages: []*template.Template{message}}}, sessionID: "session"}, + } { + t.Run(tc.name, func(t *testing.T) { + definition := &domain.PromptDefinition{ID: "prompt", SessionID: tc.sessionID, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "hello"}}} + if _, err := NewCompiledRenderer(tc.lookup).Render(context.Background(), definition, nil, nil); !errors.Is(err, ErrRenderFailure) { + t.Fatalf("render error = %v, want render failure", err) + } + }) + } +} + type compiledRendererTestLookup struct { id string templates CompiledTemplates } +func capturedTestRenderer(t testing.TB, definition *domain.PromptDefinition) Renderer { + t.Helper() + var compiled CompiledTemplates + var err error + if definition.SessionID != "" { + compiled.Session, err = CompileTemplate("session", definition.SessionID) + if err != nil { + t.Fatal(err) + } + } + for _, message := range definition.Templates { + parsed, err := CompileTemplate("message", message.Content) + if err != nil { + t.Fatal(err) + } + compiled.Messages = append(compiled.Messages, parsed) + } + return NewCompiledRenderer(compiledRendererTestLookup{id: definition.ID, templates: compiled}) +} + func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (CompiledTemplates, bool) { if id != lookup.id { return CompiledTemplates{}, false @@ -571,8 +610,7 @@ func (lookup compiledRendererTestLookup) GetCompiledTemplates(id string) (Compil return lookup.templates, true } -// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call -// template-parsing fallback rather than the captured compiled path. +// BenchmarkGoRendererArtifactReferences measures rendering captured templates. func BenchmarkGoRendererArtifactReferences(b *testing.B) { body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content ")) inputs := map[string]*domain.Artifact{ @@ -604,7 +642,7 @@ func BenchmarkGoRendererArtifactReferences(b *testing.B) { for _, tc := range tests { b.Run(tc.name, func(b *testing.B) { - renderer := NewGoRenderer() + renderer := capturedTestRenderer(b, tc.definition) b.ReportAllocs() b.SetBytes(int64(len(body))) for range b.N { diff --git a/internal/prompt/requirements_test.go b/internal/prompt/requirements_test.go index 737a1da..635f32d 100644 --- a/internal/prompt/requirements_test.go +++ b/internal/prompt/requirements_test.go @@ -314,7 +314,7 @@ func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) { "active": {Name: "active", Body: []byte("active")}, "inactive": {Name: "inactive", Body: []byte("inactive")}, } - rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars) + rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars) if err != nil { t.Fatal(err) } @@ -526,7 +526,7 @@ func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) { t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables) } artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}} - rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars) + rendered, err := capturedTestRenderer(t, definition).Render(context.Background(), definition, artifacts, test.vars) if err != nil { t.Fatal(err) } diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index 07d3c01..d335574 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -13,6 +13,7 @@ import ( "strings" "sync" "testing" + "text/template" "time" "go.maximumdirect.net/promptkit/internal/capacity" @@ -41,6 +42,36 @@ type staticPromptRepo struct { def *domain.PromptDefinition } +type capturedTemplateFixture struct { + id string + templates prompt.CompiledTemplates +} + +func (fixture capturedTemplateFixture) GetCompiledTemplates(id string) (prompt.CompiledTemplates, bool) { + return fixture.templates, id == fixture.id +} + +func testCompiledRenderer(t *testing.T, definition *domain.PromptDefinition) prompt.Renderer { + t.Helper() + var templates prompt.CompiledTemplates + if definition.SessionID != "" { + var err error + templates.Session, err = prompt.CompileTemplate("session", definition.SessionID) + if err != nil { + t.Fatal(err) + } + } + templates.Messages = make([]*template.Template, 0, len(definition.Templates)) + for _, message := range definition.Templates { + compiled, err := prompt.CompileTemplate("message", message.Content) + if err != nil { + t.Fatal(err) + } + templates.Messages = append(templates.Messages, compiled) + } + return prompt.NewCompiledRenderer(capturedTemplateFixture{id: definition.ID, templates: templates}) +} + func (r staticPromptRepo) GetPromptSnapshot( context.Context, string, @@ -449,7 +480,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), - prompt.NewGoRenderer(), + testCompiledRenderer(t, def), &fakeLLM{forbid: true}, nil, nil) @@ -496,7 +527,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), - prompt.NewGoRenderer(), + testCompiledRenderer(t, def), &fakeLLM{forbid: true}, nil, nil) @@ -526,7 +557,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), - prompt.NewGoRenderer(), + testCompiledRenderer(t, def), &fakeLLM{forbid: true}, nil, nil) @@ -886,7 +917,7 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) { &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), - prompt.NewGoRenderer(), + testCompiledRenderer(t, def), &fakeLLM{forbid: true}, nil, nil) @@ -913,7 +944,7 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) { &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), - prompt.NewGoRenderer(), + testCompiledRenderer(t, def), &fakeLLM{forbid: true}, nil, nil)