package llm import ( "context" "errors" "os" "path/filepath" "strings" "sync/atomic" "testing" "testing/fstest" configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/promptkit" ) func TestPromptKitOperatorDocumentComposesForInspectionAndExecution(t *testing.T) { for _, name := range []string{ "NOTARIUS_TEST_BACKEND_KEY", "NOTARIUS_TEST_PROFILE_KEY", "NOTARIUS_TEST_CATALOG_KEY", } { t.Setenv(name, "") } operatorDir := t.TempDir() profilePath := filepath.Join(operatorDir, "operator.yaml") if err := os.WriteFile(filepath.Join(operatorDir, "inherited.yaml"), []byte(`profiles: inherited-base: backend: custom-backend model: imported-model temperature: 0.7 service_tier: flex extra_params: inherited: retained nested: inherited: true `), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(profilePath, []byte(`configkit: imports: - path: inherited.yaml backends: custom-backend: endpoint: http://127.0.0.1:1/v1 api_key_env: NOTARIUS_TEST_BACKEND_KEY concurrency_limit: 1 queue_capacity: 0 openrouter: api_key_env: NOTARIUS_TEST_CATALOG_KEY profiles: custom-profile: extends: inherited-base model: operator-model temperature: 0 api_key_required: false service_tier: null extra_params: null empty-map-profile: extends: inherited-base extra_params: {} api_key_required: false profile-key-profile: backend: custom-backend model: profile-key-model api_key_env: NOTARIUS_TEST_PROFILE_KEY api_key_required: false required-profile: extends: inherited-base api_key_env: NOTARIUS_TEST_PROFILE_KEY api_key_required: true catalog-profile: backend: openrouter model: openai/gpt-5.6-luna api_key_required: false prompts: adapter.test: v1: output: repair_attempts: 1 `), 0o600); err != nil { t.Fatal(err) } assets := newTestPromptKitAssets(t) fake := &fakePromptKitLLM{} client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: assets, ProfileFile: profilePath, EngineOptions: []promptkit.Option{ promptkit.WithLLMClient(fake), }, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v", err) } inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ Source: PromptKitProfileSourceConfig{ProfileFile: profilePath}, Assets: assets, }) if err != nil { t.Fatalf("NewPromptKitProfileInspector() error = %v", err) } for _, test := range []struct { id string backend string model string keyEnv string required bool }{ {id: "custom-profile", backend: "custom-backend", model: "operator-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"}, {id: "inherited-base", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"}, {id: "empty-map-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_BACKEND_KEY"}, {id: "profile-key-profile", backend: "custom-backend", model: "profile-key-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY"}, {id: "required-profile", backend: "custom-backend", model: "imported-model", keyEnv: "NOTARIUS_TEST_PROFILE_KEY", required: true}, {id: "catalog-profile", backend: promptkit.BackendOpenRouter, model: "openai/gpt-5.6-luna", keyEnv: "NOTARIUS_TEST_CATALOG_KEY"}, } { inspection, err := inspector.InspectProfile(context.Background(), test.id) if err != nil { t.Fatalf("InspectProfile(%q) error = %v", test.id, err) } if inspection.ProfileID != test.id || inspection.BackendID != test.backend || inspection.Model != test.model || inspection.CredentialEnvironment != test.keyEnv || inspection.CredentialRequired != test.required { t.Fatalf("InspectProfile(%q) = %#v", test.id, inspection) } } var out map[string]any response, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", ProfileID: "custom-profile", SessionID: "source-composition-test", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err != nil { t.Fatalf("CompleteStructured() with an unset optional credential error = %v", err) } if response.Debug == nil || response.Debug.Prompt == nil { t.Fatalf("completion debug = %#v, want captured prompt", response.Debug) } debug := response.Debug.Prompt if debug.SelectedBackendID != "custom-backend" || debug.SelectedProfileID != "custom-profile" { t.Fatalf("runtime selection = backend %q, profile %q", debug.SelectedBackendID, debug.SelectedProfileID) } if got := debug.EffectiveModelParams["model"]; got != "operator-model" { t.Fatalf("runtime model = %#v, want operator-model", got) } if got, ok := debug.EffectiveModelParams["temperature"].(float64); !ok || got != 0 { t.Fatalf("runtime temperature = %#v, want explicit zero", debug.EffectiveModelParams["temperature"]) } if got := debug.EffectiveModelParams["service_tier"]; got == "flex" { t.Fatalf("explicit null retained inherited service tier: %#v", got) } if got, ok := debug.EffectiveModelParams["extra_params"]; ok && got != nil { t.Fatalf("explicit null retained inherited provider map: %#v", got) } if got := debug.OutputContract["repair_attempts"]; got != float64(1) { t.Fatalf("overlaid output repair attempts = %#v, want 1", got) } if len(debug.Messages) == 0 || !strings.Contains(debug.Messages[0].Content, "Transcript:") { t.Fatalf("rendered prompt messages = %#v", debug.Messages) } if got := atomic.LoadInt32(&fake.calls); got != 1 { t.Fatalf("provider calls after optional-credential execution = %d, want 1", got) } _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", ProfileID: "required-profile", SessionID: "source-composition-test", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err == nil || !strings.Contains(err.Error(), "api_key_env points to an unset environment variable") { t.Fatalf("CompleteStructured() with required unset credential error = %v, want native missing-environment failure", err) } if got := atomic.LoadInt32(&fake.calls); got != 1 { t.Fatalf("provider calls after required-credential failure = %d, want 1", got) } _, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ PromptID: "adapter.test", ProfileID: "empty-map-profile", SessionID: "source-composition-test", Inputs: contracts.LLMInputSet{ "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), }, }, &out) if err != nil { t.Fatalf("CompleteStructured() with empty overlay map error = %v", err) } if got := fake.lastRequest().Target.ExtraParams["inherited"]; got != "retained" { t.Fatalf("empty provider-map overlay result = %#v, want inherited entry", fake.lastRequest().Target.ExtraParams) } } func TestPromptKitPromptContentUsesDeclaringAndReplacementSourceContexts(t *testing.T) { rootDirectory := t.TempDir() declaringDirectory := filepath.Join(rootDirectory, "child") replacementDirectory := t.TempDir() writeFile := func(path, content string) { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) } } rootConfig := filepath.Join(rootDirectory, "operator.yaml") declaringConfig := filepath.Join(declaringDirectory, "imported.yaml") replacementConfig := filepath.Join(replacementDirectory, "replacement.yaml") writeFile(rootConfig, `configkit: imports: - path: child/imported.yaml `) writeFile(filepath.Join(rootDirectory, "messages", "shared.tmpl"), "root decoy") writeFile(declaringConfig, `prompts: imported.capture: v1: messages: - role: user content_file: messages/shared.tmpl output: format: text validation_mode: none `) writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import") writeFile(replacementConfig, `prompts: adapter.test: v1: messages: - role: user content_file: messages/shared.tmpl `) writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement original") endpoint, model := "https://example.invalid/v1", "content-context-model" newClient := func() *PromptKitClient { t.Helper() client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("content context profile", map[string]promptkit.ProfileConfig{ "content-context-profile": {Endpoint: &endpoint, Model: &model}, })}}}, {Inputs: []configkit.Input{{Source: configkit.File(rootConfig), Imports: true}}}, {Inputs: []configkit.Input{{Source: configkit.File(replacementConfig), Imports: true}}}, }}, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v", err) } return client } prepareMessage := func(client *PromptKitClient, promptID string) string { t.Helper() request := promptkit.RunRequest{PromptID: promptID, ProfileID: "content-context-profile", SessionID: "content-context-session"} if promptID == "adapter.test" { request.Inputs = map[string]promptkit.ArtifactRef{ "transcript": promptkit.Inline(`{"source":true}`), } } prepared, err := client.engine.Prepare(context.Background(), request) if err != nil { t.Fatalf("Prepare(%q) error = %v", promptID, err) } if len(prepared.Messages) != 1 { t.Fatalf("Prepare(%q) messages = %#v, want one", promptID, prepared.Messages) } return prepared.Messages[0].Content } first := newClient() if got := prepareMessage(first, "imported.capture"); got != "declared import" { t.Fatalf("imported prompt content = %q, want declaring-source content", got) } if got := prepareMessage(first, "adapter.test"); got != "replacement original" { t.Fatalf("partially overlaid application prompt content = %q, want replacement-source content", got) } writeFile(filepath.Join(declaringDirectory, "messages", "shared.tmpl"), "declared import updated") writeFile(filepath.Join(replacementDirectory, "messages", "shared.tmpl"), "replacement updated") if got := prepareMessage(first, "imported.capture"); got != "declared import" { t.Fatalf("existing engine imported prompt content after edit = %q", got) } if got := prepareMessage(first, "adapter.test"); got != "replacement original" { t.Fatalf("existing engine replacement content after edit = %q", got) } second := newClient() if got := prepareMessage(second, "imported.capture"); got != "declared import updated" { t.Fatalf("new engine imported prompt content = %q", got) } if got := prepareMessage(second, "adapter.test"); got != "replacement updated" { t.Fatalf("new engine replacement content = %q", got) } } func TestPromptKitStaticMessageFailuresOccurAtConstruction(t *testing.T) { for _, test := range []struct { name string prompt string files fstest.MapFS }{ { name: "missing message file", prompt: `prompts: construction.failure: v1: messages: - role: user content_file: messages/missing.tmpl output: format: text validation_mode: none `, }, { name: "invalid static template", prompt: `prompts: construction.failure: v1: messages: - role: user content: '{{ .request_value' output: format: text validation_mode: none `, }, } { t.Run(test.name, func(t *testing.T) { assets := NewAssetRegistry() files := test.files if files == nil { files = fstest.MapFS{} } files["prompt.yaml"] = &fstest.MapFile{Data: []byte(test.prompt)} if err := assets.RegisterPromptFS(files, "."); err != nil { t.Fatal(err) } _, err := NewPromptKitClient(PromptKitClientConfig{ Assets: assets, Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"construction-profile": { Endpoint: promptKitTestString("https://example.invalid/v1"), Model: promptKitTestString("construction-model"), }}), }) if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("NewPromptKitClient() error = %v, want static construction failure", err) } }) } } func TestPromptKitMissingRequestVariableFailsDuringPreparation(t *testing.T) { assets := NewAssetRegistry() if err := assets.RegisterPromptFS(fstest.MapFS{ "prompt.yaml": {Data: []byte(`prompts: request.variable: v1: messages: - role: user content: 'Value: {{ .request_value }}' output: format: text validation_mode: none `)}, }, "."); err != nil { t.Fatal(err) } client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: assets, Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"request-profile": { Endpoint: promptKitTestString("https://example.invalid/v1"), Model: promptKitTestString("request-model"), }}), }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want valid static template", err) } if _, err := client.engine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: "request.variable", ProfileID: "request-profile", }); !errors.Is(err, promptkit.ErrPromptRender) { t.Fatalf("Prepare() error = %v, want missing request variable render failure", err) } } func TestPromptKitProfileSourceGroupsPreserveNativeLayerRules(t *testing.T) { t.Run("directory group conflicts", func(t *testing.T) { directory := t.TempDir() for name, model := range map[string]string{"first.yaml": "first-model", "second.yaml": "second-model"} { content := "profiles:\n duplicate:\n endpoint: http://127.0.0.1:1/v1\n model: " + model + "\n" if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0o600); err != nil { t.Fatal(err) } } _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ Source: PromptKitProfileSourceConfig{ProfileDir: directory}, }) if err == nil { t.Fatal("NewPromptKitProfileInspector() error = nil, want same-group conflict") } }) t.Run("later group repairs an invalid lower field and overlays supplied values", func(t *testing.T) { badEndpoint := "not an endpoint" goodEndpoint := "http://127.0.0.1:1/v1" lowerModel := "lower-model" sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("lower source", map[string]promptkit.ProfileConfig{ "layered-profile": {Endpoint: &badEndpoint, Model: &lowerModel}, })}}}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("upper source", map[string]promptkit.ProfileConfig{ "layered-profile": {Endpoint: &goodEndpoint}, })}}}, }} inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources}) if err != nil { t.Fatalf("NewPromptKitProfileInspector() error = %v, want repaired final layer", err) } inspection, err := inspector.InspectProfile(context.Background(), "layered-profile") if err != nil || inspection.Model != lowerModel { t.Fatalf("InspectProfile() = (%#v, %v), want lower model retained", inspection, err) } }) t.Run("empty and missing selected directories", func(t *testing.T) { if _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ Source: PromptKitProfileSourceConfig{ProfileDir: filepath.Join(t.TempDir(), "missing")}, }); err == nil { t.Fatal("missing selected directory was accepted") } empty := t.TempDir() inspector, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{ Source: PromptKitProfileSourceConfig{ProfileDir: empty}, }) if err != nil { t.Fatalf("empty selected directory error = %v, want no contribution", err) } if _, err := inspector.InspectProfile(context.Background(), "not-present"); err == nil { t.Fatal("InspectProfile() in empty directory error = nil, want unknown profile") } }) t.Run("unused invalid final profile fails construction", func(t *testing.T) { badEndpoint := "not an endpoint" model := "unused-model" sources := promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{ Source: promptkit.ProfileSource("unused invalid profile", map[string]promptkit.ProfileConfig{ "unused-profile": {Endpoint: &badEndpoint, Model: &model}, }), }}}}} _, err := NewPromptKitProfileInspector(PromptKitProfileInspectorConfig{Sources: sources}) if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("NewPromptKitProfileInspector() error = %v, want invalid final configuration", err) } }) }