From c2e2b74f8338d7cb4de81e18f7e9f94278dc4347 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 4 Oct 2026 12:30:38 +0000 Subject: [PATCH] Protect PromptKit source and definition semantics --- docs/roadmap/implementation.md | 16 +- .../adapter/cli/framework_sources_test.go | 378 ++++++++++++++++++ 2 files changed, 391 insertions(+), 3 deletions(-) create mode 100644 internal/adapter/cli/framework_sources_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index fdb098c..7bf9543 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -3,8 +3,8 @@ ## Status, Baseline, And Authority Planned on 2026-10-04 against Scriptorium -`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–7 are complete; stages -8–13 are pending. +`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–8 are complete; stages +9–13 are pending. This plan is for a Luna coding agent: implement the stages in sequential order, normally one stage per implementation prompt. The [feature roadmap](promptkit.md) owns the target state, scope, and confirmed @@ -466,7 +466,17 @@ contracts and shared operational state preserved. ## Stage 8: Protect Native Source And Definition Semantics At The App Boundary -Status: Pending. Depends on Stage 7. +Status: Complete. Focused CLI-boundary tests now protect same-group conflicts, +ordered overlays, empty sources, invalid lower values replaced before final +validation, malformed and invalid unused definitions, and combined framework +documents. Imported prompt content retains its source-relative resource origin +and construction-time capture while schema and request artifacts remain +preparation-time reads. Native backend/profile inheritance, null and empty-map +semantics, explicit numeric presence at the injected-client boundary, partial +profiles completed by request overrides, and literal whitespace/slash IDs are +covered. Existing full-feature adoption fixtures remain in place. Focused +adoption tests, `go test ./...`, `go test -race ./...`, `go vet ./...`, the CLI +build, `gofmt`, and `git diff --check` pass. Files: a focused `internal/adapter/cli/framework_sources_test.go` and existing adoption tests where they already own the behavior. Reuse migrated helpers. diff --git a/internal/adapter/cli/framework_sources_test.go b/internal/adapter/cli/framework_sources_test.go new file mode 100644 index 0000000..4316b55 --- /dev/null +++ b/internal/adapter/cli/framework_sources_test.go @@ -0,0 +1,378 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" + appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" +) + +func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) { + conflicting := t.TempDir() + writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles: + selected: + model: first +`) + writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles: + selected: + model: second +`) + if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil { + t.Fatal("conflicting files in one source group unexpectedly constructed an engine") + } + + lower := t.TempDir() + upper := t.TempDir() + empty := t.TempDir() + writeFrameworkSourceFile(t, lower, "lower.yml", `profiles: + selected: + endpoint: not-a-url + model: lower-model + temperature: 10 +`) + writeFrameworkSourceFile(t, upper, "upper.yml", `profiles: + selected: + endpoint: http://127.0.0.1:1/v1 + model: upper-model + temperature: 0.5 +`) + engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{ + {Directory: lower}, {Directory: empty}, {Directory: upper}, + }, "") + inspection, err := engine.InspectProfile(context.Background(), "selected") + if err != nil { + t.Fatalf("inspect profile after later source replacement: %v", err) + } + if inspection.EffectiveModelParams.Endpoint != "http://127.0.0.1:1/v1" || inspection.EffectiveModelParams.Model != "upper-model" || inspection.EffectiveModelParams.Temperature != 0.5 { + t.Fatalf("later source did not replace lower values before validation: %+v", inspection.EffectiveModelParams) + } +} + +func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources(t *testing.T) { + root := t.TempDir() + combined := filepath.Join(root, "combined.yml") + writeFrameworkSourceFile(t, root, "combined.yml", `backends: + local: + endpoint: http://127.0.0.1:1/v1 +profiles: + selected: + backend: local + model: combined-model +prompts: + combined.prompt: + "1": + default_profile: selected + messages: + - role: user + content: hello + output: + format: text + validation_mode: none +`) + client := &frameworkSourceCaptureClient{} + engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client)) + if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt", PromptVersion: "1"}); err != nil { + t.Fatalf("run definitions from one combined document: %v", err) + } + if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" { + t.Fatalf("combined backend/profile/prompt values did not reach generation: %+v", client.requests) + } + + malformed := filepath.Join(root, "malformed.yml") + writeFrameworkSourceFile(t, root, "malformed.yml", "not: [valid\n") + if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: malformed}}}); err == nil { + t.Fatal("malformed framework source unexpectedly constructed an engine") + } + + invalidUnused := filepath.Join(root, "invalid-unused.yml") + writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts: + unused.prompt: + "1": + messages: + - role: user + content: "{{" + output: + format: text + validation_mode: none +`) + if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err) + } +} + +func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testing.T) { + root := t.TempDir() + catalogDir := filepath.Join(root, "catalog") + if err := os.MkdirAll(catalogDir, 0o755); err != nil { + t.Fatal(err) + } + schemaDir := filepath.Join(root, "schemas") + if err := os.MkdirAll(schemaDir, 0o755); err != nil { + t.Fatal(err) + } + mainFile := filepath.Join(root, "main.yml") + writeFrameworkSourceFile(t, root, "main.yml", `configkit: + imports: + - path: catalog/base.yml +prompts: + " team/prompt ": + " derived/v2 ": + extends: " base/v1 " +`) + writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles: + " profile/primary ": + endpoint: http://127.0.0.1:1/v1 + model: imported-model +prompts: + " team/prompt ": + " base/v1 ": + default_profile: " profile/primary " + inputs: + - name: source + required: true + messages: + - role: user + content_file: ./body.md + - role: user + content: 'Artifact: {{input "source"}}' + output: + format: json + validation_mode: json_schema + schema_path: response.schema.json +`) + writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}") + artifactPath := filepath.Join(root, "request-artifact.txt") + writeFrameworkSourceFile(t, root, "request-artifact.txt", "initial artifact") + writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"initial schema"}`) + selections := []appconfig.SourceSelection{{File: mainFile}} + engine := newFrameworkSourceTestEngine(t, selections, schemaDir) + + writeFrameworkSourceFile(t, catalogDir, "body.md", "edited message for {{.audience}}") + writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact") + writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`) + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: " team/prompt ", PromptVersion: " derived/v2 ", + Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)}, + Vars: map[string]string{"audience": "reader"}, + }) + if err != nil { + t.Fatalf("prepare inherited imported prompt: %v", err) + } + if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" { + t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages) + } + if prepared.SelectedProfileID != " profile/primary " || prepared.PromptVersion != " derived/v2 " { + t.Fatalf("literal profile/version keys were not preserved: %+v", prepared) + } + if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" { + t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput) + } + + newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir) + newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: " team/prompt ", PromptVersion: " derived/v2 ", + Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)}, + Vars: map[string]string{"audience": "reader"}, + }) + if err != nil { + t.Fatalf("prepare with newly captured message: %v", err) + } + if newPrepared.Messages[0].Content != "edited message for reader" { + t.Fatalf("new engine did not observe the message edit: %+v", newPrepared.Messages[0]) + } + + configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir)) + code, stdout, stderr := runCLICommand(t, renderCommand, []string{ + "--config", configPath, + "--prompt", " team/prompt ", + "--prompt-version", " derived/v2 ", + "--input", "source=" + artifactPath, + "--var", "audience=reader", + "--format", "json", + }) + if code != ExitOK { + t.Fatalf("render exact literal IDs through CLI: code=%d stderr=%q", code, stderr) + } + var rendered promptkit.PreparedRun + if err := json.Unmarshal([]byte(stdout), &rendered); err != nil { + t.Fatalf("decode rendered literal-ID fixture: %v", err) + } + if rendered.PromptID != " team/prompt " || rendered.PromptVersion != " derived/v2 " || rendered.SelectedProfileID != " profile/primary " { + t.Fatalf("CLI changed exact prompt/profile/version keys: %+v", rendered) + } +} + +func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T) { + root := t.TempDir() + frameworkFile := filepath.Join(root, "framework.yml") + writeFrameworkSourceFile(t, root, "framework.yml", `backends: + first: + endpoint: https://first.example/v1 + api_key_env: FIRST_BACKEND_KEY + extra_params: + source: first + nested: + first_backend: true + second: + endpoint: https://second.example/v1 + api_key_env: SECOND_BACKEND_KEY + extra_params: + source: second + nested: + second_backend: true +profiles: + parent: + backend: first + model: parent-model + temperature: 0.7 + top_p: 0.6 + max_tokens: 45 + timeout_seconds: 30 + service_tier: flex + extra_params: + nested: + parent_profile: true + retained: "yes" + backend-child: + extends: parent + backend: second + model: child-model + presence-child: + extends: parent + backend: second + api_key_env: null + temperature: null + max_tokens: 0 + timeout_seconds: 0 + extra_params: + nested: + explicit_false: false + map-null: + backend: second + model: map-model + extra_params: null + map-empty: + backend: second + model: map-model + extra_params: {} +prompts: + execution.prompt: + "1": + messages: + - role: user + content: hello + output: + format: text + validation_mode: none +`) + client := &frameworkSourceCaptureClient{} + engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client)) + + for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} { + if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", PromptVersion: "1", ProfileID: profileID}); err != nil { + t.Fatalf("run profile %q: %v", profileID, err) + } + } + if len(client.requests) != 4 { + t.Fatalf("expected four provider-bound requests, got %d", len(client.requests)) + } + + backendChild := client.requests[0].Target + wantBackendParams := map[string]any{"source": "second", "nested": map[string]any{ + "second_backend": true, "parent_profile": true, "retained": "yes", + }} + if backendChild.BackendID != "second" || backendChild.Endpoint != "https://second.example/v1" || backendChild.APIKeyEnv != "SECOND_BACKEND_KEY" || backendChild.Model != "child-model" || + backendChild.Temperature != 0.7 || backendChild.TopP != 0.6 || backendChild.MaxTokens != 45 || backendChild.TimeoutSeconds != 30 || backendChild.ServiceTier != "flex" || !reflect.DeepEqual(backendChild.ExtraParams, wantBackendParams) { + t.Fatalf("backend change lost backend-derived or explicit inherited values: %+v", backendChild) + } + + presenceRequest := client.requests[1] + presenceTarget := presenceRequest.Target + if presenceTarget.APIKeyEnv != "" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 || + !presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP { + t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence) + } + wantPresenceParams := map[string]any{"source": "second", "nested": map[string]any{ + "second_backend": true, "parent_profile": true, "retained": "yes", "explicit_false": false, + }} + if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) { + t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams) + } + if client.requests[2].Target.ExtraParams != nil { + t.Fatalf("whole-map null did not suppress backend defaults: %#v", client.requests[2].Target.ExtraParams) + } + if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) { + t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams) + } +} + +func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) { + root := t.TempDir() + frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles: + partial: {} +prompts: + partial.prompt: + "1": + default_profile: partial + messages: + - role: user + content: hello + output: + format: text + validation_mode: none +`) + provider := newTestLLMServer("completed partial profile", nil) + defer provider.Close() + configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) + code, stdout, stderr := runCLICommand(t, runCommand, []string{ + "--config", configPath, + "--prompt", "partial.prompt", + "--llm-base-url", provider.URL + "/v1", + "--model", "request-model", + }) + if code != ExitOK || stdout != "completed partial profile" { + t.Fatalf("request overrides did not complete the partial profile: code=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func newFrameworkSourceTestEngine(t *testing.T, selections []appconfig.SourceSelection, schemaDir string, options ...promptkit.Option) *promptkit.Engine { + t.Helper() + engine, err := newEngine(engineSettings{promptkitSources: selections, schemaDir: schemaDir}, options...) + if err != nil { + var loadError *configkit.LoadError + if errors.As(err, &loadError) { + t.Fatalf("construct engine from selected framework sources: %v (%+v)", err, loadError.Diagnostics()) + } + t.Fatalf("construct engine from selected framework sources: %v", err) + } + return engine +} + +func writeFrameworkSourceFile(t *testing.T, directory, name, contents string) string { + t.Helper() + if err := os.MkdirAll(directory, 0o755); err != nil { + t.Fatalf("create framework source directory: %v", err) + } + path := filepath.Join(directory, name) + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatalf("write framework source %q: %v", path, err) + } + return path +} + +type frameworkSourceCaptureClient struct { + requests []promptkit.GenerateRequest +} + +func (client *frameworkSourceCaptureClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.requests = append(client.requests, request) + return &promptkit.GenerateResponse{Content: "ok"}, nil +}