package cli import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "testing" configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config" ) func TestFrameworkSourceGroupsComposeAndValidateEffectiveDefinitions(t *testing.T) { conflicting := t.TempDir() writeFrameworkSourceFile(t, conflicting, "first.yml", `profiles: selected: model: first `) writeFrameworkSourceFile(t, conflicting, "second.yml", `profiles: selected: model: second `) if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{Directory: conflicting}}}); err == nil { t.Fatal("conflicting files in one source group unexpectedly constructed an engine") } lower := t.TempDir() upper := t.TempDir() empty := t.TempDir() writeFrameworkSourceFile(t, lower, "lower.yml", `profiles: selected: endpoint: not-a-url model: lower-model temperature: 10 `) writeFrameworkSourceFile(t, upper, "upper.yml", `profiles: selected: endpoint: http://127.0.0.1:1/v1 model: upper-model temperature: 0.5 `) engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{ {Directory: lower}, {Directory: empty}, {Directory: upper}, }, "") inspection, err := engine.InspectProfile(context.Background(), "selected") if err != nil { t.Fatalf("inspect profile after later source replacement: %v", err) } if inspection.EffectiveModelParams.Endpoint != "http://127.0.0.1:1/v1" || inspection.EffectiveModelParams.Model != "upper-model" || inspection.EffectiveModelParams.Temperature != 0.5 { t.Fatalf("later source did not replace lower values before validation: %+v", inspection.EffectiveModelParams) } } func TestFrameworkSourceConstructionUsesCombinedDocumentAndRejectsInvalidSources(t *testing.T) { root := t.TempDir() combined := filepath.Join(root, "combined.yml") writeFrameworkSourceFile(t, root, "combined.yml", `backends: local: endpoint: http://127.0.0.1:1/v1 profiles: selected: backend: local model: combined-model prompts: combined.prompt: "1": default_profile: selected messages: - role: user content: hello output: format: text validation_mode: none `) client := &frameworkSourceCaptureClient{} engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: combined}}, "", promptkit.WithLLMClient(client)) if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "combined.prompt", PromptVersion: "1"}); err != nil { t.Fatalf("run definitions from one combined document: %v", err) } if len(client.requests) != 1 || client.requests[0].Target.BackendID != "local" || client.requests[0].Target.Model != "combined-model" { t.Fatalf("combined backend/profile/prompt values did not reach generation: %+v", client.requests) } malformed := filepath.Join(root, "malformed.yml") writeFrameworkSourceFile(t, root, "malformed.yml", "not: [valid\n") if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: malformed}}}); err == nil { t.Fatal("malformed framework source unexpectedly constructed an engine") } invalidUnused := filepath.Join(root, "invalid-unused.yml") writeFrameworkSourceFile(t, root, "invalid-unused.yml", `prompts: unused.prompt: "1": messages: - role: user content: "{{" output: format: text validation_mode: none `) if _, err := newEngine(engineSettings{promptkitSources: []appconfig.SourceSelection{{File: invalidUnused}}}); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("invalid unused definition did not fail as framework configuration: %v", err) } } func TestFrameworkImportsRetainResourceOriginsAndCaptureAtConstruction(t *testing.T) { root := t.TempDir() catalogDir := filepath.Join(root, "catalog") if err := os.MkdirAll(catalogDir, 0o755); err != nil { t.Fatal(err) } schemaDir := filepath.Join(root, "schemas") if err := os.MkdirAll(schemaDir, 0o755); err != nil { t.Fatal(err) } mainFile := filepath.Join(root, "main.yml") writeFrameworkSourceFile(t, root, "main.yml", `configkit: imports: - path: catalog/base.yml prompts: " team/prompt ": " derived/v2 ": extends: " base/v1 " `) writeFrameworkSourceFile(t, catalogDir, "base.yml", `profiles: " profile/primary ": endpoint: http://127.0.0.1:1/v1 model: imported-model prompts: " team/prompt ": " base/v1 ": default_profile: " profile/primary " inputs: - name: source required: true messages: - role: user content_file: ./body.md - role: user content: 'Artifact: {{input "source"}}' output: format: json validation_mode: json_schema schema_path: response.schema.json `) writeFrameworkSourceFile(t, catalogDir, "body.md", "Captured message for {{.audience}}") artifactPath := filepath.Join(root, "request-artifact.txt") writeFrameworkSourceFile(t, root, "request-artifact.txt", "initial artifact") writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"initial schema"}`) selections := []appconfig.SourceSelection{{File: mainFile}} engine := newFrameworkSourceTestEngine(t, selections, schemaDir) writeFrameworkSourceFile(t, catalogDir, "body.md", "edited message for {{.audience}}") writeFrameworkSourceFile(t, root, "request-artifact.txt", "prepared artifact") writeFrameworkSourceFile(t, schemaDir, "response.schema.json", `{"type":"object","title":"prepared schema"}`) prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: " team/prompt ", PromptVersion: " derived/v2 ", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)}, Vars: map[string]string{"audience": "reader"}, }) if err != nil { t.Fatalf("prepare inherited imported prompt: %v", err) } if len(prepared.Messages) != 2 || prepared.Messages[0].Content != "Captured message for reader" || prepared.Messages[1].Content != "Artifact: prepared artifact" { t.Fatalf("resource origin or capture/preparation behavior changed: %+v", prepared.Messages) } if prepared.SelectedProfileID != " profile/primary " || prepared.PromptVersion != " derived/v2 " { t.Fatalf("literal profile/version keys were not preserved: %+v", prepared) } if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil || prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] != "prepared schema" { t.Fatalf("schema was not loaded during preparation: %+v", prepared.StructuredOutput) } newEngine := newFrameworkSourceTestEngine(t, selections, schemaDir) newPrepared, err := newEngine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: " team/prompt ", PromptVersion: " derived/v2 ", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.File(artifactPath)}, Vars: map[string]string{"audience": "reader"}, }) if err != nil { t.Fatalf("prepare with newly captured message: %v", err) } if newPrepared.Messages[0].Content != "edited message for reader" { t.Fatalf("new engine did not observe the message edit: %+v", newPrepared.Messages[0]) } configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\nschema_dir: %q\n", mainFile, schemaDir)) code, stdout, stderr := runCLICommand(t, renderCommand, []string{ "--config", configPath, "--prompt", " team/prompt ", "--prompt-version", " derived/v2 ", "--input", "source=" + artifactPath, "--var", "audience=reader", "--format", "json", }) if code != ExitOK { t.Fatalf("render exact literal IDs through CLI: code=%d stderr=%q", code, stderr) } var rendered promptkit.PreparedRun if err := json.Unmarshal([]byte(stdout), &rendered); err != nil { t.Fatalf("decode rendered literal-ID fixture: %v", err) } if rendered.PromptID != " team/prompt " || rendered.PromptVersion != " derived/v2 " || rendered.SelectedProfileID != " profile/primary " { t.Fatalf("CLI changed exact prompt/profile/version keys: %+v", rendered) } } func TestNativeProfileInheritanceAndPresenceReachTheClientBoundary(t *testing.T) { root := t.TempDir() frameworkFile := filepath.Join(root, "framework.yml") writeFrameworkSourceFile(t, root, "framework.yml", `backends: first: endpoint: https://first.example/v1 api_key_env: FIRST_BACKEND_KEY extra_params: source: first nested: first_backend: true second: endpoint: https://second.example/v1 api_key_env: SECOND_BACKEND_KEY extra_params: source: second nested: second_backend: true profiles: parent: backend: first model: parent-model temperature: 0.7 top_p: 0.6 max_tokens: 45 timeout_seconds: 30 service_tier: flex extra_params: nested: parent_profile: true retained: "yes" backend-child: extends: parent backend: second model: child-model presence-child: extends: parent backend: second api_key_env: null temperature: null max_tokens: 0 timeout_seconds: 0 extra_params: nested: explicit_false: false map-null: backend: second model: map-model extra_params: null map-empty: backend: second model: map-model extra_params: {} prompts: execution.prompt: "1": messages: - role: user content: hello output: format: text validation_mode: none `) client := &frameworkSourceCaptureClient{} engine := newFrameworkSourceTestEngine(t, []appconfig.SourceSelection{{File: frameworkFile}}, "", promptkit.WithLLMClient(client)) for _, profileID := range []string{"backend-child", "presence-child", "map-null", "map-empty"} { if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "execution.prompt", PromptVersion: "1", ProfileID: profileID}); err != nil { t.Fatalf("run profile %q: %v", profileID, err) } } if len(client.requests) != 4 { t.Fatalf("expected four provider-bound requests, got %d", len(client.requests)) } backendChild := client.requests[0].Target wantBackendParams := map[string]any{"source": "second", "nested": map[string]any{ "second_backend": true, "parent_profile": true, "retained": "yes", }} if backendChild.BackendID != "second" || backendChild.Endpoint != "https://second.example/v1" || backendChild.APIKeyEnv != "SECOND_BACKEND_KEY" || backendChild.Model != "child-model" || backendChild.Temperature != 0.7 || backendChild.TopP != 0.6 || backendChild.MaxTokens != 45 || backendChild.TimeoutSeconds != 30 || backendChild.ServiceTier != "flex" || !reflect.DeepEqual(backendChild.ExtraParams, wantBackendParams) { t.Fatalf("backend change lost backend-derived or explicit inherited values: %+v", backendChild) } presenceRequest := client.requests[1] presenceTarget := presenceRequest.Target if presenceTarget.APIKeyEnv != "" || presenceTarget.Temperature != 0 || presenceTarget.MaxTokens != 0 || presenceTarget.TimeoutSeconds != 0 || !presenceRequest.TargetPresence.MaxTokens || !presenceRequest.TargetPresence.TimeoutSeconds || presenceRequest.TargetPresence.Temperature || !presenceRequest.TargetPresence.TopP { t.Fatalf("explicit null/zero presence did not reach the injected client: target=%+v presence=%+v", presenceTarget, presenceRequest.TargetPresence) } wantPresenceParams := map[string]any{"source": "second", "nested": map[string]any{ "second_backend": true, "parent_profile": true, "retained": "yes", "explicit_false": false, }} if !reflect.DeepEqual(presenceTarget.ExtraParams, wantPresenceParams) { t.Fatalf("recursive provider-map retention changed: %#v", presenceTarget.ExtraParams) } if client.requests[2].Target.ExtraParams != nil { t.Fatalf("whole-map null did not suppress backend defaults: %#v", client.requests[2].Target.ExtraParams) } if !reflect.DeepEqual(client.requests[3].Target.ExtraParams, map[string]any{"source": "second", "nested": map[string]any{"second_backend": true}}) { t.Fatalf("empty provider map unexpectedly cleared backend defaults: %#v", client.requests[3].Target.ExtraParams) } } func TestCLICompletesPartialFrameworkProfileWithRequestOverrides(t *testing.T) { root := t.TempDir() frameworkFile := writeFrameworkSourceFile(t, root, "partial.yml", `profiles: partial: {} prompts: partial.prompt: "1": default_profile: partial messages: - role: user content: hello output: format: text validation_mode: none `) provider := newTestLLMServer("completed partial profile", nil) defer provider.Close() configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile)) code, stdout, stderr := runCLICommand(t, runCommand, []string{ "--config", configPath, "--prompt", "partial.prompt", "--llm-base-url", provider.URL + "/v1", "--model", "request-model", }) if code != ExitOK || stdout != "completed partial profile" { t.Fatalf("request overrides did not complete the partial profile: code=%d stdout=%q stderr=%q", code, stdout, stderr) } } func newFrameworkSourceTestEngine(t *testing.T, selections []appconfig.SourceSelection, schemaDir string, options ...promptkit.Option) *promptkit.Engine { t.Helper() engine, err := newEngine(engineSettings{promptkitSources: selections, schemaDir: schemaDir}, options...) if err != nil { var loadError *configkit.LoadError if errors.As(err, &loadError) { t.Fatalf("construct engine from selected framework sources: %v (%+v)", err, loadError.Diagnostics()) } t.Fatalf("construct engine from selected framework sources: %v", err) } return engine } func writeFrameworkSourceFile(t *testing.T, directory, name, contents string) string { t.Helper() if err := os.MkdirAll(directory, 0o755); err != nil { t.Fatalf("create framework source directory: %v", err) } path := filepath.Join(directory, name) if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { t.Fatalf("write framework source %q: %v", path, err) } return path } type frameworkSourceCaptureClient struct { requests []promptkit.GenerateRequest } func (client *frameworkSourceCaptureClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.requests = append(client.requests, request) return &promptkit.GenerateResponse{Content: "ok"}, nil }