Protect PromptKit source and definition semantics
This commit is contained in:
@@ -3,8 +3,8 @@
|
||||
## Status, Baseline, And Authority
|
||||
|
||||
Planned on 2026-10-04 against Scriptorium
|
||||
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–7 are complete; stages
|
||||
8–13 are pending.
|
||||
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–8 are complete; stages
|
||||
9–13 are pending.
|
||||
This plan is for a Luna coding agent: implement the stages in sequential order,
|
||||
normally one stage per implementation prompt. The
|
||||
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
||||
@@ -466,7 +466,17 @@ contracts and shared operational state preserved.
|
||||
|
||||
## Stage 8: Protect Native Source And Definition Semantics At The App Boundary
|
||||
|
||||
Status: Pending. Depends on Stage 7.
|
||||
Status: Complete. Focused CLI-boundary tests now protect same-group conflicts,
|
||||
ordered overlays, empty sources, invalid lower values replaced before final
|
||||
validation, malformed and invalid unused definitions, and combined framework
|
||||
documents. Imported prompt content retains its source-relative resource origin
|
||||
and construction-time capture while schema and request artifacts remain
|
||||
preparation-time reads. Native backend/profile inheritance, null and empty-map
|
||||
semantics, explicit numeric presence at the injected-client boundary, partial
|
||||
profiles completed by request overrides, and literal whitespace/slash IDs are
|
||||
covered. Existing full-feature adoption fixtures remain in place. Focused
|
||||
adoption tests, `go test ./...`, `go test -race ./...`, `go vet ./...`, the CLI
|
||||
build, `gofmt`, and `git diff --check` pass.
|
||||
|
||||
Files: a focused `internal/adapter/cli/framework_sources_test.go` and existing
|
||||
adoption tests where they already own the behavior. Reuse migrated helpers.
|
||||
|
||||
378
internal/adapter/cli/framework_sources_test.go
Normal file
378
internal/adapter/cli/framework_sources_test.go
Normal file
@@ -0,0 +1,378 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||
)
|
||||
|
||||
func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) {
|
||||
conflicting := t.TempDir()
|
||||
writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles:
|
||||
selected:
|
||||
model: first
|
||||
`)
|
||||
writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles:
|
||||
selected:
|
||||
model: second
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil {
|
||||
t.Fatal("conflicting files in one source group unexpectedly constructed an engine")
|
||||
}
|
||||
|
||||
lower := t.TempDir()
|
||||
upper := t.TempDir()
|
||||
empty := t.TempDir()
|
||||
writeFrameworkSourceFile(t, lower, "lower.yml", `profiles:
|
||||
selected:
|
||||
endpoint: not-a-url
|
||||
model: lower-model
|
||||
temperature: 10
|
||||
`)
|
||||
writeFrameworkSourceFile(t, upper, "upper.yml", `profiles:
|
||||
selected:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: upper-model
|
||||
temperature: 0.5
|
||||
`)
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{
|
||||
{Directory: lower}, {Directory: empty}, {Directory: upper},
|
||||
}, "")
|
||||
inspection, err := engine.InspectProfile(context.Background(), "selected")
|
||||
if err != nil {
|
||||
t.Fatalf("inspect profile after later source replacement: %v", err)
|
||||
}
|
||||
if inspection.EffectiveModelParams.Endpoint != "http://127.0.0.1:1/v1" || inspection.EffectiveModelParams.Model != "upper-model" || inspection.EffectiveModelParams.Temperature != 0.5 {
|
||||
t.Fatalf("later source did not replace lower values before validation: %+v", inspection.EffectiveModelParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
combined := filepath.Join(root, "combined.yml")
|
||||
writeFrameworkSourceFile(t, root, "combined.yml", `backends:
|
||||
local:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
profiles:
|
||||
selected:
|
||||
backend: local
|
||||
model: combined-model
|
||||
prompts:
|
||||
combined.prompt:
|
||||
"1":
|
||||
default_profile: selected
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
client := &frameworkSourceCaptureClient{}
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client))
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt", PromptVersion: "1"}); err != nil {
|
||||
t.Fatalf("run definitions from one combined document: %v", err)
|
||||
}
|
||||
if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" {
|
||||
t.Fatalf("combined backend/profile/prompt values did not reach generation: %+v", client.requests)
|
||||
}
|
||||
|
||||
malformed := filepath.Join(root, "malformed.yml")
|
||||
writeFrameworkSourceFile(t, root, "malformed.yml", "not: [valid\n")
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: malformed}}}); err == nil {
|
||||
t.Fatal("malformed framework source unexpectedly constructed an engine")
|
||||
}
|
||||
|
||||
invalidUnused := filepath.Join(root, "invalid-unused.yml")
|
||||
writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts:
|
||||
unused.prompt:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
catalogDir := filepath.Join(root, "catalog")
|
||||
if err := os.MkdirAll(catalogDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schemaDir := filepath.Join(root, "schemas")
|
||||
if err := os.MkdirAll(schemaDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mainFile := filepath.Join(root, "main.yml")
|
||||
writeFrameworkSourceFile(t, root, "main.yml", `configkit:
|
||||
imports:
|
||||
- path: catalog/base.yml
|
||||
prompts:
|
||||
" team/prompt ":
|
||||
" derived/v2 ":
|
||||
extends: " base/v1 "
|
||||
`)
|
||||
writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles:
|
||||
" profile/primary ":
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
model: imported-model
|
||||
prompts:
|
||||
" team/prompt ":
|
||||
" base/v1 ":
|
||||
default_profile: " profile/primary "
|
||||
inputs:
|
||||
- name: source
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./body.md
|
||||
- role: user
|
||||
content: 'Artifact: {{input "source"}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: response.schema.json
|
||||
`)
|
||||
writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}")
|
||||
artifactPath := filepath.Join(root, "request-artifact.txt")
|
||||
writeFrameworkSourceFile(t, root, "request-artifact.txt", "initial artifact")
|
||||
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"initial schema"}`)
|
||||
selections := []appconfig.SourceSelection{{File: mainFile}}
|
||||
engine := newFrameworkSourceTestEngine(t, selections, schemaDir)
|
||||
|
||||
writeFrameworkSourceFile(t, catalogDir, "body.md", "edited message for {{.audience}}")
|
||||
writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact")
|
||||
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`)
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare inherited imported prompt: %v", err)
|
||||
}
|
||||
if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" {
|
||||
t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages)
|
||||
}
|
||||
if prepared.SelectedProfileID != " profile/primary " || prepared.PromptVersion != " derived/v2 " {
|
||||
t.Fatalf("literal profile/version keys were not preserved: %+v", prepared)
|
||||
}
|
||||
if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" {
|
||||
t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput)
|
||||
}
|
||||
|
||||
newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir)
|
||||
newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
|
||||
Vars: map[string]string{"audience": "reader"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with newly captured message: %v", err)
|
||||
}
|
||||
if newPrepared.Messages[0].Content != "edited message for reader" {
|
||||
t.Fatalf("new engine did not observe the message edit: %+v", newPrepared.Messages[0])
|
||||
}
|
||||
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir))
|
||||
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", " team/prompt ",
|
||||
"--prompt-version", " derived/v2 ",
|
||||
"--input", "source=" + artifactPath,
|
||||
"--var", "audience=reader",
|
||||
"--format", "json",
|
||||
})
|
||||
if code != ExitOK {
|
||||
t.Fatalf("render exact literal IDs through CLI: code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
var rendered promptkit.PreparedRun
|
||||
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
|
||||
t.Fatalf("decode rendered literal-ID fixture: %v", err)
|
||||
}
|
||||
if rendered.PromptID != " team/prompt " || rendered.PromptVersion != " derived/v2 " || rendered.SelectedProfileID != " profile/primary " {
|
||||
t.Fatalf("CLI changed exact prompt/profile/version keys: %+v", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := filepath.Join(root, "framework.yml")
|
||||
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||
first:
|
||||
endpoint: https://first.example/v1
|
||||
api_key_env: FIRST_BACKEND_KEY
|
||||
extra_params:
|
||||
source: first
|
||||
nested:
|
||||
first_backend: true
|
||||
second:
|
||||
endpoint: https://second.example/v1
|
||||
api_key_env: SECOND_BACKEND_KEY
|
||||
extra_params:
|
||||
source: second
|
||||
nested:
|
||||
second_backend: true
|
||||
profiles:
|
||||
parent:
|
||||
backend: first
|
||||
model: parent-model
|
||||
temperature: 0.7
|
||||
top_p: 0.6
|
||||
max_tokens: 45
|
||||
timeout_seconds: 30
|
||||
service_tier: flex
|
||||
extra_params:
|
||||
nested:
|
||||
parent_profile: true
|
||||
retained: "yes"
|
||||
backend-child:
|
||||
extends: parent
|
||||
backend: second
|
||||
model: child-model
|
||||
presence-child:
|
||||
extends: parent
|
||||
backend: second
|
||||
api_key_env: null
|
||||
temperature: null
|
||||
max_tokens: 0
|
||||
timeout_seconds: 0
|
||||
extra_params:
|
||||
nested:
|
||||
explicit_false: false
|
||||
map-null:
|
||||
backend: second
|
||||
model: map-model
|
||||
extra_params: null
|
||||
map-empty:
|
||||
backend: second
|
||||
model: map-model
|
||||
extra_params: {}
|
||||
prompts:
|
||||
execution.prompt:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
client := &frameworkSourceCaptureClient{}
|
||||
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client))
|
||||
|
||||
for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} {
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", PromptVersion: "1", ProfileID: profileID}); err != nil {
|
||||
t.Fatalf("run profile %q: %v", profileID, err)
|
||||
}
|
||||
}
|
||||
if len(client.requests) != 4 {
|
||||
t.Fatalf("expected four provider-bound requests, got %d", len(client.requests))
|
||||
}
|
||||
|
||||
backendChild := client.requests[0].Target
|
||||
wantBackendParams := map[string]any{"source": "second", "nested": map[string]any{
|
||||
"second_backend": true, "parent_profile": true, "retained": "yes",
|
||||
}}
|
||||
if backendChild.BackendID != "second" || backendChild.Endpoint != "https://second.example/v1" || backendChild.APIKeyEnv != "SECOND_BACKEND_KEY" || backendChild.Model != "child-model" ||
|
||||
backendChild.Temperature != 0.7 || backendChild.TopP != 0.6 || backendChild.MaxTokens != 45 || backendChild.TimeoutSeconds != 30 || backendChild.ServiceTier != "flex" || !reflect.DeepEqual(backendChild.ExtraParams, wantBackendParams) {
|
||||
t.Fatalf("backend change lost backend-derived or explicit inherited values: %+v", backendChild)
|
||||
}
|
||||
|
||||
presenceRequest := client.requests[1]
|
||||
presenceTarget := presenceRequest.Target
|
||||
if presenceTarget.APIKeyEnv != "" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
|
||||
!presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP {
|
||||
t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence)
|
||||
}
|
||||
wantPresenceParams := map[string]any{"source": "second", "nested": map[string]any{
|
||||
"second_backend": true, "parent_profile": true, "retained": "yes", "explicit_false": false,
|
||||
}}
|
||||
if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) {
|
||||
t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams)
|
||||
}
|
||||
if client.requests[2].Target.ExtraParams != nil {
|
||||
t.Fatalf("whole-map null did not suppress backend defaults: %#v", client.requests[2].Target.ExtraParams)
|
||||
}
|
||||
if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
|
||||
t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles:
|
||||
partial: {}
|
||||
prompts:
|
||||
partial.prompt:
|
||||
"1":
|
||||
default_profile: partial
|
||||
messages:
|
||||
- role: user
|
||||
content: hello
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
provider := newTestLLMServer("completed partial profile", nil)
|
||||
defer provider.Close()
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||
code, stdout, stderr := runCLICommand(t, runCommand, []string{
|
||||
"--config", configPath,
|
||||
"--prompt", "partial.prompt",
|
||||
"--llm-base-url", provider.URL + "/v1",
|
||||
"--model", "request-model",
|
||||
})
|
||||
if code != ExitOK || stdout != "completed partial profile" {
|
||||
t.Fatalf("request overrides did not complete the partial profile: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func newFrameworkSourceTestEngine(t *testing.T, selections []appconfig.SourceSelection, schemaDir string, options ...promptkit.Option) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := newEngine(engineSettings{promptkitSources: selections, schemaDir: schemaDir}, options...)
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
t.Fatalf("construct engine from selected framework sources: %v (%+v)", err, loadError.Diagnostics())
|
||||
}
|
||||
t.Fatalf("construct engine from selected framework sources: %v", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func writeFrameworkSourceFile(t *testing.T, directory, name, contents string) string {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
t.Fatalf("create framework source directory: %v", err)
|
||||
}
|
||||
path := filepath.Join(directory, name)
|
||||
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
|
||||
t.Fatalf("write framework source %q: %v", path, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
type frameworkSourceCaptureClient struct {
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (client *frameworkSourceCaptureClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.requests = append(client.requests, request)
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user