Files
scriptorium/internal/adapter/cli/framework_sources_test.go

379 lines
14 KiB
Go

package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
)
func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) {
conflicting := t.TempDir()
writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles:
selected:
model: first
`)
writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles:
selected:
model: second
`)
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil {
t.Fatal("conflicting files in one source group unexpectedly constructed an engine")
}
lower := t.TempDir()
upper := t.TempDir()
empty := t.TempDir()
writeFrameworkSourceFile(t, lower, "lower.yml", `profiles:
selected:
endpoint: not-a-url
model: lower-model
temperature: 10
`)
writeFrameworkSourceFile(t, upper, "upper.yml", `profiles:
selected:
endpoint: http://127.0.0.1:1/v1
model: upper-model
temperature: 0.5
`)
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{
{Directory: lower}, {Directory: empty}, {Directory: upper},
}, "")
inspection, err := engine.InspectProfile(context.Background(), "selected")
if err != nil {
t.Fatalf("inspect profile after later source replacement: %v", err)
}
if inspection.EffectiveModelParams.Endpoint != "http://127.0.0.1:1/v1" || inspection.EffectiveModelParams.Model != "upper-model" || inspection.EffectiveModelParams.Temperature != 0.5 {
t.Fatalf("later source did not replace lower values before validation: %+v", inspection.EffectiveModelParams)
}
}
func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources(t *testing.T) {
root := t.TempDir()
combined := filepath.Join(root, "combined.yml")
writeFrameworkSourceFile(t, root, "combined.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
profiles:
selected:
backend: local
model: combined-model
prompts:
combined.prompt:
"1":
default_profile: selected
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
client := &frameworkSourceCaptureClient{}
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client))
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt", PromptVersion: "1"}); err != nil {
t.Fatalf("run definitions from one combined document: %v", err)
}
if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" {
t.Fatalf("combined backend/profile/prompt values did not reach generation: %+v", client.requests)
}
malformed := filepath.Join(root, "malformed.yml")
writeFrameworkSourceFile(t, root, "malformed.yml", "not: [valid\n")
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: malformed}}}); err == nil {
t.Fatal("malformed framework source unexpectedly constructed an engine")
}
invalidUnused := filepath.Join(root, "invalid-unused.yml")
writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts:
unused.prompt:
"1":
messages:
- role: user
content: "{{"
output:
format: text
validation_mode: none
`)
if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err)
}
}
func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testing.T) {
root := t.TempDir()
catalogDir := filepath.Join(root, "catalog")
if err := os.MkdirAll(catalogDir, 0o755); err != nil {
t.Fatal(err)
}
schemaDir := filepath.Join(root, "schemas")
if err := os.MkdirAll(schemaDir, 0o755); err != nil {
t.Fatal(err)
}
mainFile := filepath.Join(root, "main.yml")
writeFrameworkSourceFile(t, root, "main.yml", `configkit:
imports:
- path: catalog/base.yml
prompts:
" team/prompt ":
" derived/v2 ":
extends: " base/v1 "
`)
writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles:
" profile/primary ":
endpoint: http://127.0.0.1:1/v1
model: imported-model
prompts:
" team/prompt ":
" base/v1 ":
default_profile: " profile/primary "
inputs:
- name: source
required: true
messages:
- role: user
content_file: ./body.md
- role: user
content: 'Artifact: {{input "source"}}'
output:
format: json
validation_mode: json_schema
schema_path: response.schema.json
`)
writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}")
artifactPath := filepath.Join(root, "request-artifact.txt")
writeFrameworkSourceFile(t, root, "request-artifact.txt", "initial artifact")
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"initial schema"}`)
selections := []appconfig.SourceSelection{{File: mainFile}}
engine := newFrameworkSourceTestEngine(t, selections, schemaDir)
writeFrameworkSourceFile(t, catalogDir, "body.md", "edited message for {{.audience}}")
writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact")
writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
Vars: map[string]string{"audience": "reader"},
})
if err != nil {
t.Fatalf("prepare inherited imported prompt: %v", err)
}
if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" {
t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages)
}
if prepared.SelectedProfileID != " profile/primary " || prepared.PromptVersion != " derived/v2 " {
t.Fatalf("literal profile/version keys were not preserved: %+v", prepared)
}
if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" {
t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput)
}
newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir)
newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: " team/prompt ", PromptVersion: " derived/v2 ",
Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)},
Vars: map[string]string{"audience": "reader"},
})
if err != nil {
t.Fatalf("prepare with newly captured message: %v", err)
}
if newPrepared.Messages[0].Content != "edited message for reader" {
t.Fatalf("new engine did not observe the message edit: %+v", newPrepared.Messages[0])
}
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir))
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
"--config", configPath,
"--prompt", " team/prompt ",
"--prompt-version", " derived/v2 ",
"--input", "source=" + artifactPath,
"--var", "audience=reader",
"--format", "json",
})
if code != ExitOK {
t.Fatalf("render exact literal IDs through CLI: code=%d stderr=%q", code, stderr)
}
var rendered promptkit.PreparedRun
if err := json.Unmarshal([]byte(stdout), &rendered); err != nil {
t.Fatalf("decode rendered literal-ID fixture: %v", err)
}
if rendered.PromptID != " team/prompt " || rendered.PromptVersion != " derived/v2 " || rendered.SelectedProfileID != " profile/primary " {
t.Fatalf("CLI changed exact prompt/profile/version keys: %+v", rendered)
}
}
func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T) {
root := t.TempDir()
frameworkFile := filepath.Join(root, "framework.yml")
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
first:
endpoint: https://first.example/v1
api_key_env: FIRST_BACKEND_KEY
extra_params:
source: first
nested:
first_backend: true
second:
endpoint: https://second.example/v1
api_key_env: SECOND_BACKEND_KEY
extra_params:
source: second
nested:
second_backend: true
profiles:
parent:
backend: first
model: parent-model
temperature: 0.7
top_p: 0.6
max_tokens: 45
timeout_seconds: 30
service_tier: flex
extra_params:
nested:
parent_profile: true
retained: "yes"
backend-child:
extends: parent
backend: second
model: child-model
presence-child:
extends: parent
backend: second
api_key_env: null
temperature: null
max_tokens: 0
timeout_seconds: 0
extra_params:
nested:
explicit_false: false
map-null:
backend: second
model: map-model
extra_params: null
map-empty:
backend: second
model: map-model
extra_params: {}
prompts:
execution.prompt:
"1":
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
client := &frameworkSourceCaptureClient{}
engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client))
for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} {
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", PromptVersion: "1", ProfileID: profileID}); err != nil {
t.Fatalf("run profile %q: %v", profileID, err)
}
}
if len(client.requests) != 4 {
t.Fatalf("expected four provider-bound requests, got %d", len(client.requests))
}
backendChild := client.requests[0].Target
wantBackendParams := map[string]any{"source": "second", "nested": map[string]any{
"second_backend": true, "parent_profile": true, "retained": "yes",
}}
if backendChild.BackendID != "second" || backendChild.Endpoint != "https://second.example/v1" || backendChild.APIKeyEnv != "SECOND_BACKEND_KEY" || backendChild.Model != "child-model" ||
backendChild.Temperature != 0.7 || backendChild.TopP != 0.6 || backendChild.MaxTokens != 45 || backendChild.TimeoutSeconds != 30 || backendChild.ServiceTier != "flex" || !reflect.DeepEqual(backendChild.ExtraParams, wantBackendParams) {
t.Fatalf("backend change lost backend-derived or explicit inherited values: %+v", backendChild)
}
presenceRequest := client.requests[1]
presenceTarget := presenceRequest.Target
if presenceTarget.APIKeyEnv != "" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 ||
!presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP {
t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence)
}
wantPresenceParams := map[string]any{"source": "second", "nested": map[string]any{
"second_backend": true, "parent_profile": true, "retained": "yes", "explicit_false": false,
}}
if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) {
t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams)
}
if client.requests[2].Target.ExtraParams != nil {
t.Fatalf("whole-map null did not suppress backend defaults: %#v", client.requests[2].Target.ExtraParams)
}
if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) {
t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams)
}
}
func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) {
root := t.TempDir()
frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles:
partial: {}
prompts:
partial.prompt:
"1":
default_profile: partial
messages:
- role: user
content: hello
output:
format: text
validation_mode: none
`)
provider := newTestLLMServer("completed partial profile", nil)
defer provider.Close()
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
code, stdout, stderr := runCLICommand(t, runCommand, []string{
"--config", configPath,
"--prompt", "partial.prompt",
"--llm-base-url", provider.URL + "/v1",
"--model", "request-model",
})
if code != ExitOK || stdout != "completed partial profile" {
t.Fatalf("request overrides did not complete the partial profile: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
}
func newFrameworkSourceTestEngine(t *testing.T, selections []appconfig.SourceSelection, schemaDir string, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
engine, err := newEngine(engineSettings{promptkitSources: selections, schemaDir: schemaDir}, options...)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("construct engine from selected framework sources: %v (%+v)", err, loadError.Diagnostics())
}
t.Fatalf("construct engine from selected framework sources: %v", err)
}
return engine
}
func writeFrameworkSourceFile(t *testing.T, directory, name, contents string) string {
t.Helper()
if err := os.MkdirAll(directory, 0o755); err != nil {
t.Fatalf("create framework source directory: %v", err)
}
path := filepath.Join(directory, name)
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatalf("write framework source %q: %v", path, err)
}
return path
}
type frameworkSourceCaptureClient struct {
requests []promptkit.GenerateRequest
}
func (client *frameworkSourceCaptureClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.requests = append(client.requests, request)
return &promptkit.GenerateResponse{Content: "ok"}, nil
}