diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index cdefda4f..f90748d9 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -292,6 +292,10 @@ Prompt size: constructor/error call sites only. ## Stage 4 — Migrate LLM Fixtures And Restore The Adapter Suite +> Completion: complete. `GOWORK=off go test ./internal/framework/llm` passes +> offline; legacy keyed-format and removed-API fixtures are cleared from the +> LLM adapter suite. CLI and registered-module fixtures remain for Stage 5. + Goal: exercise released configuration through existing adapter coverage. Inspect `promptkit_client_test.go`, `promptkit_observation_test.go`, diff --git a/internal/framework/llm/asset_registry_test.go b/internal/framework/llm/asset_registry_test.go index bbe45de1..79c5c668 100644 --- a/internal/framework/llm/asset_registry_test.go +++ b/internal/framework/llm/asset_registry_test.go @@ -2,12 +2,14 @@ package llm import ( "context" + "errors" "io/fs" "strings" "testing" "testing/fstest" "time" + configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" ) @@ -18,7 +20,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) { "prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)}, "prompts/messages/task.tmpl": {Data: []byte("Return JSON.")}, "schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)}, - }, "prompts") + }, ".") mustRegisterSchemaFS(t, registry, fstest.MapFS{ "root/schemas/out.json": {Data: []byte(`{"type":"object"}`)}, }, "root") @@ -42,7 +44,7 @@ func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) { } } -func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) { +func TestAssetRegistryConstructionFailsForMissingPromptAsset(t *testing.T) { registry := NewAssetRegistry() mustRegisterPromptFS(t, registry, fstest.MapFS{ "test.yaml": {Data: []byte(validPromptYAML("out.json"))}, @@ -51,16 +53,21 @@ func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) { "out.json": {Data: []byte(`{"type":"object"}`)}, }, ".") - engine := newPromptKitAssetTestEngine(t, registry) - _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "asset.test", - ProfileID: "asset-test-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline(`{"ok":true}`), - }, - }) - if err == nil || !strings.Contains(err.Error(), "content_file") { - t.Fatalf("Prepare() error = %v, want missing content_file error", err) + sources, options, err := registry.PromptKitConfigurationSources() + if err != nil { + t.Fatal(err) + } + sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{ + Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{ + "asset-test-profile": { + Endpoint: promptKitTestString("http://127.0.0.1:1/v1"), + Model: promptKitTestString("asset-test-model"), + }, + }), + }}}) + _, err = promptkit.NewEngine(promptkit.Config{Sources: sources}, options...) + if err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "messages/user.tmpl") { + t.Fatalf("NewEngine() error = %v, want invalid configuration for missing prompt content", err) } } @@ -75,8 +82,23 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) { "present.json": {Data: []byte(`{"type":"object"}`)}, }, ".") - engine := newPromptKitAssetTestEngine(t, registry) - _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + sources, options, err := registry.PromptKitConfigurationSources() + if err != nil { + t.Fatal(err) + } + sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{ + Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{ + "asset-test-profile": { + Endpoint: promptKitTestString("http://127.0.0.1:1/v1"), + Model: promptKitTestString("asset-test-model"), + }, + }), + }}}) + engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ PromptID: "asset.test", ProfileID: "asset-test-profile", Inputs: map[string]promptkit.ArtifactRef{ @@ -84,14 +106,14 @@ func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) { }, }) if err == nil || !strings.Contains(err.Error(), "missing.json") { - t.Fatalf("Prepare() error = %v, want missing schema error", err) + t.Fatalf("Prepare() error = %v, want missing schema asset failure", err) } } func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) { registry := NewAssetRegistry() - mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one") - mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two") + mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("prompts: {}")}}, "one") + mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("prompts: {}")}}, "two") _, err := registry.PromptFS() if err == nil || !strings.Contains(err.Error(), "duplicate asset path") { @@ -102,12 +124,12 @@ func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) { func TestAssetRegistryCombinesFallbackProfileSources(t *testing.T) { registry := NewAssetRegistry() if err := registry.RegisterFallbackProfileFS(fstest.MapFS{ - "first/profiles/one.yaml": {Data: []byte("id: one\nmodel: first\n")}, + "first/profiles/one.yaml": {Data: []byte("profiles:\n one:\n model: first\n")}, }, "first/profiles"); err != nil { t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err) } if err := registry.RegisterFallbackProfileFS(fstest.MapFS{ - "second/two.yaml": {Data: []byte("id: two\nmodel: second\n")}, + "second/two.yaml": {Data: []byte("profiles:\n two:\n model: second\n")}, }, "second"); err != nil { t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err) } @@ -144,10 +166,10 @@ func TestAssetRegistryRejectsUnreadableFallbackProfileAssets(t *testing.T) { func TestAssetRegistryRejectsDuplicateFallbackProfilePaths(t *testing.T) { registry := NewAssetRegistry() - if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("id: first\n")}}, "first"); err != nil { + if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"first/profile.yaml": {Data: []byte("profiles:\n first: {}\n")}}, "first"); err != nil { t.Fatal(err) } - if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("id: second\n")}}, "second"); err != nil { + if err := registry.RegisterFallbackProfileFS(fstest.MapFS{"second/profile.yaml": {Data: []byte("profiles:\n second: {}\n")}}, "second"); err != nil { t.Fatal(err) } _, err := registry.FallbackProfileFS() @@ -172,8 +194,8 @@ func TestAssetRegistryFallbackProfileDigestTracksContentWithoutLeakingIt(t *test return digest } - first := digestFor("id: fallback\nmodel: model-one\n") - second := digestFor("id: fallback\nmodel: model-two\n") + first := digestFor("profiles:\n fallback:\n model: model-one\n") + second := digestFor("profiles:\n fallback:\n model: model-two\n") if first == second { t.Fatalf("fallback profile digests = %q and %q, want content change", first, second) } @@ -231,16 +253,19 @@ func TestHashAssetsOmitsRawAssetContent(t *testing.T) { func newPromptKitAssetTestEngine(t *testing.T, registry *AssetRegistry) *promptkit.Engine { t.Helper() - options, err := registry.PromptKitOptions() + sources, options, err := registry.PromptKitConfigurationSources() if err != nil { - t.Fatalf("PromptKitOptions() error = %v, want nil", err) + t.Fatalf("PromptKitConfigurationSources() error = %v, want nil", err) } - options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "asset-test-profile", - Endpoint: "http://127.0.0.1:1/v1", - Model: "asset-test-model", - }))) - engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...) + sources.Underlays = append(sources.Underlays, configkit.Underlay{Inputs: []configkit.Input{{ + Source: promptkit.ProfileSource("asset test profile", map[string]promptkit.ProfileConfig{ + "asset-test-profile": { + Endpoint: promptKitTestString("http://127.0.0.1:1/v1"), + Model: promptKitTestString("asset-test-model"), + }, + }), + }}}) + engine, err := promptkit.NewEngine(promptkit.Config{Sources: sources, Timeout: time.Second}, options...) if err != nil { t.Fatalf("NewEngine() error = %v, want nil", err) } @@ -262,22 +287,23 @@ func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.Map } func validPromptYAML(schemaPath string) string { - return `id: asset.test -version: "v1" -inputs: - - name: transcript - required: true - content_type: application/json -messages: - - role: user - content_file: ./messages/user.tmpl - - role: user - content_file: ./messages/task.tmpl -output: - format: json - validation_mode: json_schema - schema_path: ` + schemaPath + ` - repair_attempts: 0 + return `prompts: + asset.test: + v1: + inputs: + - name: transcript + required: true + content_type: application/json + messages: + - role: user + content_file: ./messages/user.tmpl + - role: user + content_file: ./messages/task.tmpl + output: + format: json + validation_mode: json_schema + schema_path: ` + schemaPath + ` + repair_attempts: 0 ` } diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index 54cf900a..055f4020 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -28,6 +28,7 @@ type PromptKitLocalBackendConfig struct { type PromptKitClientConfig struct { ProfileDir string ProfileFile string + Sources promptkit.ConfigurationSources TransientRetries *int LocalBackend *PromptKitLocalBackendConfig Assets *AssetRegistry @@ -76,7 +77,7 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) { ProfileDir: cfg.ProfileDir, ProfileFile: cfg.ProfileFile, LocalBackend: cfg.LocalBackend, - }, cfg.Assets) + }, cfg.Assets, cfg.Sources) if err != nil { return nil, &promptKitClientConfigurationError{err: err} } diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index b4056cf7..eb85d6d5 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -190,94 +190,74 @@ func TestPromptKitClientRejectsInvalidCorrectionsBeforePromptPreparation(t *test } func TestPromptKitClientUsesOnePreparedSnapshotForDebugAndGeneration(t *testing.T) { - const initialPrompt = `id: snapshot.test -version: "v1" -default_profile: snapshot-profile -inputs: - - name: transcript - required: true - content_type: application/json -messages: - - role: user - content: "Snapshot A: {{ input \"transcript\" }}" -output: - format: json - validation_mode: json_schema - schema_path: adapter.schema.json - repair_attempts: 0 + const initialPrompt = `prompts: + snapshot.test: + v1: + default_profile: snapshot-profile + inputs: + - name: transcript + required: true + content_type: application/json + messages: + - role: user + content: "Snapshot A: {{ input \"transcript\" }}" + output: + format: json + validation_mode: json_schema + schema_path: adapter.schema.json + repair_attempts: 0 ` - const updatedPrompt = `id: snapshot.test -version: "v1" -default_profile: snapshot-profile -inputs: - - name: transcript - required: true - content_type: application/json -messages: - - role: user - content: "Snapshot B: {{ input \"transcript\" }}" -output: - format: json - validation_mode: json_schema - schema_path: adapter.schema.json - repair_attempts: 0 + const updatedPrompt = `prompts: + snapshot.test: + v1: + default_profile: snapshot-profile + inputs: + - name: transcript + required: true + content_type: application/json + messages: + - role: user + content: "Snapshot B: {{ input \"transcript\" }}" + output: + format: json + validation_mode: json_schema + schema_path: adapter.schema.json + repair_attempts: 0 ` - source := &switchingPromptFS{files: fstest.MapFS{ + source := fstest.MapFS{ "snapshot.test.yaml": {Data: []byte(initialPrompt)}, - }} + } fake := &fakePromptKitLLM{content: `{"ok":true}`} client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), - EngineOptions: []promptkit.Option{ - promptkit.WithPromptFS(source, "."), - promptkit.WithBackend(promptkit.Backend{ID: "snapshot-backend", Endpoint: "http://promptkit.test/v1"}), - promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "snapshot-profile", - BackendID: "snapshot-backend", - Model: "snapshot-model", - })), - promptkit.WithLLMClient(fake), - }, + Sources: promptKitTestSources( + map[string]promptkit.BackendConfig{"snapshot-backend": {Endpoint: promptKitTestString("http://promptkit.test/v1")}}, + map[string]promptkit.ProfileConfig{"snapshot-profile": {Backend: promptKitTestString("snapshot-backend"), Model: promptKitTestString("snapshot-model")}}, + promptKitTestInputs(t, "snapshot-prompt", source, ".")..., + ), + EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want nil", err) } - opened := source.holdNextPromptRead() - defer source.resumePromptRead() - type completion struct { - response contracts.StructuredCompletionResponse - err error + source["snapshot.test.yaml"] = &fstest.MapFile{Data: []byte(updatedPrompt)} + var out struct { + OK bool `json:"ok"` } - completed := make(chan completion, 1) - go func() { - var out struct { - OK bool `json:"ok"` - } - response, callErr := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ - PromptID: "snapshot.test", - Inputs: contracts.LLMInputSet{ - "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), - }, - }, &out) - completed <- completion{response: response, err: callErr} - }() - select { - case <-opened: - case <-time.After(time.Second): - t.Fatal("PromptKit did not read the prompt source") + result, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{ + PromptID: "snapshot.test", + Inputs: contracts.LLMInputSet{ + "transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""), + }, + }, &out) + if err != nil { + t.Fatalf("CompleteStructured() error = %v, want nil", err) } - source.replacePrompt([]byte(updatedPrompt)) - source.resumePromptRead() - - result := <-completed - if result.err != nil { - t.Fatalf("CompleteStructured() error = %v, want nil", result.err) + if result.Debug == nil || result.Debug.Prompt == nil || len(result.Debug.Prompt.Messages) != 1 { + t.Fatalf("debug prompt = %#v, want one prepared message", result.Debug) } - if result.response.Debug == nil || result.response.Debug.Prompt == nil || len(result.response.Debug.Prompt.Messages) != 1 { - t.Fatalf("debug prompt = %#v, want one prepared message", result.response.Debug) - } - debugContent := result.response.Debug.Prompt.Messages[0].Content + debugContent := result.Debug.Prompt.Messages[0].Content generatedContent := fake.lastRequest().Prompt.Messages[0].Content if debugContent != generatedContent { t.Fatalf("debug content = %q, generation content = %q, want one snapshot", debugContent, generatedContent) @@ -399,26 +379,22 @@ func TestPromptKitClientAppliesReasoningEffortOverride(t *testing.T) { } func TestNewPromptKitClientReportsAssetAndEngineConstructionFailures(t *testing.T) { - t.Run("assets", func(t *testing.T) { + t.Run("asset read", func(t *testing.T) { registry := NewAssetRegistry() - for _, root := range []string{"one", "two"} { - if err := registry.RegisterPromptFS(fstest.MapFS{ - root + "/prompt.yaml": {Data: []byte("id: duplicate")}, - }, root); err != nil { - t.Fatalf("RegisterPromptFS() error = %v", err) - } + if err := registry.RegisterPromptFS(unreadableAssetFS{}, "."); err != nil { + t.Fatalf("RegisterPromptFS() error = %v", err) } if _, err := NewPromptKitClient(PromptKitClientConfig{Assets: registry}); err == nil || - !strings.Contains(err.Error(), "duplicate asset path") { - t.Fatalf("NewPromptKitClient() error = %v, want asset construction failure", err) + !errors.Is(err, fs.ErrPermission) || strings.Contains(err.Error(), "permission denied") { + t.Fatalf("NewPromptKitClient() error = %v, want safe asset read failure", err) } }) t.Run("engine", func(t *testing.T) { if _, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), - EngineOptions: []promptkit.Option{promptkit.WithProfileFile("")}, - }); err == nil || !strings.Contains(err.Error(), "create PromptKit engine") { + EngineOptions: []promptkit.Option{promptkit.WithLLMClient(nil)}, + }); err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) || strings.Contains(err.Error(), "nil client") { t.Fatalf("NewPromptKitClient() error = %v, want engine construction failure", err) } }) @@ -429,7 +405,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { const credentialEnvironment = "PROMPTKIT_TEST_API_KEY" writeProfile := func(model string) { t.Helper() - content := "id: checkpoint-profile\nendpoint: http://promptkit.test/v1\nmodel: " + model + "\napi_key_env: " + credentialEnvironment + "\n" + content := "profiles:\n checkpoint-profile:\n endpoint: http://promptkit.test/v1\n model: " + model + "\n api_key_env: " + credentialEnvironment + "\n" if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil { t.Fatal(err) } @@ -508,7 +484,7 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { profileDir := t.TempDir() firstPath := filepath.Join(profileDir, "first-profile.yaml") secondPath := filepath.Join(profileDir, "second-profile.yaml") - content := []byte("id: directory-profile\nendpoint: http://promptkit.test/v1\nmodel: directory-model\n") + content := []byte("profiles:\n directory-profile:\n endpoint: http://promptkit.test/v1\n model: directory-model\n") if err := os.WriteFile(firstPath, content, 0o600); err != nil { t.Fatal(err) } @@ -535,17 +511,17 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { profileDir := t.TempDir() parentPath := filepath.Join(profileDir, "parent.yaml") leafPath := filepath.Join(profileDir, "leaf.yaml") - if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-one\n"), 0o600); err != nil { + if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-one\n"), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(leafPath, []byte("id: leaf\nbase_profile: parent\nmodel: leaf-model\n"), 0o600); err != nil { + if err := os.WriteFile(leafPath, []byte("profiles:\n leaf:\n extends: parent\n model: leaf-model\n"), 0o600); err != nil { t.Fatal(err) } first, err := promptKitProfileFingerprint(profileDir, "", "") if err != nil { t.Fatal(err) } - if err := os.WriteFile(parentPath, []byte("id: parent\nendpoint: http://promptkit.test/v1\nmodel: parent-two\n"), 0o600); err != nil { + if err := os.WriteFile(parentPath, []byte("profiles:\n parent:\n endpoint: http://promptkit.test/v1\n model: parent-two\n"), 0o600); err != nil { t.Fatal(err) } second, err := promptKitProfileFingerprint(profileDir, "", "") @@ -590,7 +566,7 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing assets := newTestPromptKitAssets(t) const profileID = "fallback-profile" if err := assets.RegisterFallbackProfileFS(fstest.MapFS{ - "profiles/fallback.yaml": {Data: []byte("id: " + profileID + "\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")}, + "profiles/fallback.yaml": {Data: []byte("profiles:\n " + profileID + ":\n endpoint: http://promptkit.test/v1\n model: fallback-model\n")}, }, "profiles"); err != nil { t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err) } @@ -633,10 +609,10 @@ func TestPromptKitClientUsesFallbackProfilesForExecutionAndInspection(t *testing func TestPromptKitClientUsesInheritedFilesystemProfileForInspectionAndExecution(t *testing.T) { profileDir := t.TempDir() - if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("id: base\nendpoint: http://promptkit.test/v1\nmodel: base-model\nreasoning_effort: medium\n"), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(profileDir, "base.yaml"), []byte("profiles:\n base:\n endpoint: http://promptkit.test/v1\n model: base-model\n reasoning_effort: medium\n"), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("id: inherited-profile\nbase_profile: base\nmodel: leaf-model\n"), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(profileDir, "leaf.yaml"), []byte("profiles:\n inherited-profile:\n extends: base\n model: leaf-model\n"), 0o600); err != nil { t.Fatal(err) } fake := &fakePromptKitLLM{content: `{"ok":true}`} @@ -690,7 +666,7 @@ func TestPromptKitClientAllowsMissingOptionalFilesystemCredential(t *testing.T) defer server.Close() t.Setenv("NOTARIUS_OPTIONAL_PROFILE_KEY", "") profilePath := filepath.Join(t.TempDir(), "optional.yaml") - if err := os.WriteFile(profilePath, []byte("id: optional-profile\nendpoint: "+server.URL+"/v1\nmodel: optional-model\napi_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil { + if err := os.WriteFile(profilePath, []byte("profiles:\n optional-profile:\n endpoint: "+server.URL+"/v1\n model: optional-model\n api_key_env: NOTARIUS_OPTIONAL_PROFILE_KEY\n"), 0o600); err != nil { t.Fatal(err) } client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath}) @@ -713,7 +689,7 @@ func TestPromptKitClientMapsHTTPGenerationErrorToApplicationBoundary(t *testing. defer server.Close() t.Setenv("NOTARIUS_GENERATION_TEST_KEY", credential) profilePath := filepath.Join(t.TempDir(), "profile.yaml") - if err := os.WriteFile(profilePath, []byte("id: generation-profile\nendpoint: "+server.URL+"/v1\nmodel: test\napi_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil { + if err := os.WriteFile(profilePath, []byte("profiles:\n generation-profile:\n endpoint: "+server.URL+"/v1\n model: test\n api_key_env: NOTARIUS_GENERATION_TEST_KEY\n"), 0o600); err != nil { t.Fatal(err) } client, err := NewPromptKitClient(PromptKitClientConfig{Assets: newTestPromptKitAssets(t), ProfileFile: profilePath}) @@ -778,8 +754,8 @@ func TestPromptKitClientCheckpointFingerprintTracksFallbackProfileAssets(t *test return fingerprints[0] } - first := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-one\n") - second := fingerprintFor("id: fallback\nendpoint: http://promptkit.test/v1\nmodel: model-two\n") + first := fingerprintFor("profiles:\n fallback:\n endpoint: http://promptkit.test/v1\n model: model-one\n") + second := fingerprintFor("profiles:\n fallback:\n endpoint: http://promptkit.test/v1\n model: model-two\n") if first == second { t.Fatalf("checkpoint fingerprints = %#v and %#v, want fallback asset change", first, second) } @@ -904,9 +880,10 @@ func TestPromptKitClientUsesConfiguredLocalBackend(t *testing.T) { defer server.Close() profilePath := filepath.Join(t.TempDir(), "profiles.yml") - if err := os.WriteFile(profilePath, []byte(`id: local-profile -backend: local -model: local-model + if err := os.WriteFile(profilePath, []byte(`profiles: + local-profile: + backend: local + model: local-model `), 0o600); err != nil { t.Fatal(err) } @@ -956,19 +933,11 @@ model: local-model Assets: newTestPromptKitAssets(t), ProfileFile: profilePath, }) - if err != nil { - t.Fatalf("NewPromptKitClient() without local registration error = %v, want nil", err) + if clientWithoutRegistration != nil || err == nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("NewPromptKitClient() without local registration = (%v, %v), want invalid configuration", clientWithoutRegistration, err) } - _, err = clientWithoutRegistration.CompleteStructured(context.Background(), request, &out) - if err == nil || - !strings.Contains(err.Error(), "prepare PromptKit prompt") || - !strings.Contains(err.Error(), promptkit.BackendLocal) { - t.Fatalf("CompleteStructured() without registration error = %v, want preparation failure with local backend context", err) - } - for _, sentinel := range []error{promptkit.ErrProfileLoad, promptkit.ErrInvalidRequest, promptkit.ErrPromptNotFound} { - if errors.Is(err, sentinel) { - t.Fatalf("preparation failure exposes PromptKit sentinel %v: %v", sentinel, err) - } + if strings.Contains(err.Error(), "local-profile") || strings.Contains(err.Error(), profilePath) { + t.Fatalf("configuration error exposes profile details: %v", err) } if providerCalls.Load() != 1 { t.Fatalf("provider calls after missing-registration failure = %d, want 1", providerCalls.Load()) @@ -1009,7 +978,7 @@ func TestPromptKitClientAppliesConfiguredBuiltInTransientRetries(t *testing.T) { defer server.Close() profilePath := filepath.Join(t.TempDir(), "profile.yml") - if err := os.WriteFile(profilePath, []byte("id: retry-profile\nendpoint: "+server.URL+"/v1\nmodel: retry-model\n"), 0o600); err != nil { + if err := os.WriteFile(profilePath, []byte("profiles:\n retry-profile:\n endpoint: "+server.URL+"/v1\n model: retry-model\n"), 0o600); err != nil { t.Fatal(err) } client, err := NewPromptKitClient(PromptKitClientConfig{ @@ -1232,20 +1201,16 @@ func TestPromptKitClientTranslatesBackendCapacityExhaustion(t *testing.T) { } client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), - EngineOptions: []promptkit.Option{ - promptkit.WithBackend(promptkit.Backend{ - ID: "limited-backend", - Endpoint: "http://127.0.0.1:1/v1", - ConcurrencyLimit: 1, - QueueCapacity: &queueCapacity, - }), - promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "limited-profile", - BackendID: "limited-backend", - Model: "limited-model", - })), - promptkit.WithLLMClient(fake), - }, + Sources: promptKitTestSources( + map[string]promptkit.BackendConfig{"limited-backend": { + Endpoint: promptKitTestString("http://127.0.0.1:1/v1"), + ConcurrencyLimit: promptKitTestInt(1), QueueCapacity: &queueCapacity, + }}, + map[string]promptkit.ProfileConfig{"limited-profile": { + Backend: promptKitTestString("limited-backend"), Model: promptKitTestString("limited-model"), + }}, + ), + EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want nil", err) @@ -1364,13 +1329,9 @@ func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) { Assets: newTestPromptKitAssets(t), Timeout: configuredTimeout, HTTPClient: &http.Client{Transport: transport}, - EngineOptions: []promptkit.Option{ - promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "default-profile", - Endpoint: "http://promptkit.test/v1", - Model: "default-model", - })), - }, + Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": { + Endpoint: promptKitTestString("http://promptkit.test/v1"), Model: promptKitTestString("default-model"), + }}), }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v", err) @@ -1415,9 +1376,9 @@ func TestPromptKitClientClassifiesBuiltInTransportAndMalformedResponses(t *testi client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)}, - EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model", - }))}, + Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": { + Endpoint: promptKitTestString("http://promptkit.test/v1"), Model: promptKitTestString("default-model"), + }}), }) if err != nil { t.Fatal(err) @@ -1571,27 +1532,20 @@ func newTestPromptKitClientWithReasoning(t *testing.T, fake *fakePromptKitLLM, r client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: registry, ReasoningEffort: reasoningEffort, - EngineOptions: []promptkit.Option{ - promptkit.WithBackend(promptkit.Backend{ - ID: "test-backend", - Endpoint: "http://127.0.0.1:1/v1", - }), - promptkit.WithProfiles( - promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "default-profile", - Endpoint: "http://127.0.0.1:1/v1", - Model: "default-model", - ReasoningEffort: "profile-reasoning", - }), - promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "explicit-profile", - BackendID: "test-backend", - Model: "explicit-model", - ReasoningEffort: "profile-reasoning", - }), - ), - promptkit.WithLLMClient(fake), - }, + Sources: promptKitTestSources( + map[string]promptkit.BackendConfig{"test-backend": {Endpoint: promptKitTestString("http://127.0.0.1:1/v1")}}, + map[string]promptkit.ProfileConfig{ + "default-profile": { + Endpoint: promptKitTestString("http://127.0.0.1:1/v1"), + Model: promptKitTestString("default-model"), ReasoningEffort: promptKitTestString("profile-reasoning"), + }, + "explicit-profile": { + Backend: promptKitTestString("test-backend"), + Model: promptKitTestString("explicit-model"), ReasoningEffort: promptKitTestString("profile-reasoning"), + }, + }, + ), + EngineOptions: []promptkit.Option{promptkit.WithLLMClient(fake)}, }) if err != nil { t.Fatalf("NewPromptKitClient() error = %v, want nil", err) @@ -1603,38 +1557,40 @@ func newTestPromptKitAssets(t *testing.T) *AssetRegistry { t.Helper() registry := NewAssetRegistry() if err := registry.RegisterPromptFS(fstest.MapFS{ - "adapter.test.yaml": {Data: []byte(`id: adapter.test -version: "v1" -default_profile: default-profile -session_id: "{{ .session_id }}" -inputs: - - name: transcript - required: true - content_type: application/json -messages: - - role: user - content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}" -output: - format: json - validation_mode: json_schema - schema_path: adapter.schema.json - repair_attempts: 0 + "adapter.test.yaml": {Data: []byte(`prompts: + adapter.test: + v1: + default_profile: default-profile + session_id: "{{ .session_id }}" + inputs: + - name: transcript + required: true + content_type: application/json + messages: + - role: user + content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }} Session: {{ .session_id }}" + output: + format: json + validation_mode: json_schema + schema_path: adapter.schema.json + repair_attempts: 0 `)}, - "adapter.direct-session.yaml": {Data: []byte(`id: adapter.direct-session -version: "v1" -default_profile: default-profile -inputs: - - name: transcript - required: true - content_type: application/json -messages: - - role: user - content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}" -output: - format: json - validation_mode: json_schema - schema_path: adapter.schema.json - repair_attempts: 0 + "adapter.direct-session.yaml": {Data: []byte(`prompts: + adapter.direct-session: + v1: + default_profile: default-profile + inputs: + - name: transcript + required: true + content_type: application/json + messages: + - role: user + content: "Transcript: {{ input \"transcript\" }} Custom: {{ index . \"custom\" }}" + output: + format: json + validation_mode: json_schema + schema_path: adapter.schema.json + repair_attempts: 0 `)}, }, "."); err != nil { t.Fatalf("RegisterPromptFS() error = %v", err) @@ -1647,61 +1603,6 @@ output: return registry } -type switchingPromptFS struct { - mu sync.Mutex - files fstest.MapFS - pauseNextOpen bool - opened chan struct{} - resume chan struct{} -} - -func (f *switchingPromptFS) Open(name string) (fs.File, error) { - f.mu.Lock() - file, err := f.files.Open(name) - pause := f.pauseNextOpen && name == "snapshot.test.yaml" - resume := f.resume - if pause { - f.pauseNextOpen = false - close(f.opened) - } - f.mu.Unlock() - if pause { - <-resume - } - return file, err -} - -func (f *switchingPromptFS) ReadDir(name string) ([]fs.DirEntry, error) { - f.mu.Lock() - defer f.mu.Unlock() - return f.files.ReadDir(name) -} - -func (f *switchingPromptFS) holdNextPromptRead() <-chan struct{} { - f.mu.Lock() - defer f.mu.Unlock() - f.pauseNextOpen = true - f.opened = make(chan struct{}) - f.resume = make(chan struct{}) - return f.opened -} - -func (f *switchingPromptFS) replacePrompt(content []byte) { - f.mu.Lock() - defer f.mu.Unlock() - f.files["snapshot.test.yaml"] = &fstest.MapFile{Data: append([]byte(nil), content...)} -} - -func (f *switchingPromptFS) resumePromptRead() { - f.mu.Lock() - resume := f.resume - f.resume = nil - f.mu.Unlock() - if resume != nil { - close(resume) - } -} - type fakePromptKitLLM struct { content string responses []promptkit.GenerateResponse diff --git a/internal/framework/llm/promptkit_observation_test.go b/internal/framework/llm/promptkit_observation_test.go index f68f5e8a..0f9ea826 100644 --- a/internal/framework/llm/promptkit_observation_test.go +++ b/internal/framework/llm/promptkit_observation_test.go @@ -23,9 +23,9 @@ func newObservedHTTPClient(t *testing.T, handler http.HandlerFunc, retries int, client, err := NewPromptKitClient(PromptKitClientConfig{ Assets: newTestPromptKitAssets(t), TransientRetries: &retries, HTTPClient: &http.Client{Transport: transport}, - EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ - ID: "default-profile", Endpoint: server.URL + "/v1", Model: "test-model", - }))}, + Sources: promptKitTestSources(nil, map[string]promptkit.ProfileConfig{"default-profile": { + Endpoint: promptKitTestString(server.URL + "/v1"), Model: promptKitTestString("test-model"), + }}), }) if err != nil { t.Fatal(err) diff --git a/internal/framework/llm/promptkit_profiles.go b/internal/framework/llm/promptkit_profiles.go index 22774987..955e839c 100644 --- a/internal/framework/llm/promptkit_profiles.go +++ b/internal/framework/llm/promptkit_profiles.go @@ -17,8 +17,9 @@ type PromptKitProfileSourceConfig struct { } type PromptKitProfileInspectorConfig struct { - Source PromptKitProfileSourceConfig - Assets *AssetRegistry + Source PromptKitProfileSourceConfig + Assets *AssetRegistry + Sources promptkit.ConfigurationSources } func (c PromptKitProfileSourceConfig) localEndpoint() string { @@ -75,7 +76,7 @@ func (e *promptKitProfileConfigurationError) Unwrap() error { } func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) { - _, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets) + _, sources, options, err := promptKitEngineConfiguration(cfg.Source, cfg.Assets, cfg.Sources) if err != nil { return nil, &promptKitProfileConfigurationError{err: err} } @@ -112,7 +113,7 @@ func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileI }, nil } -func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) { +func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *AssetRegistry, configuredSources promptkit.ConfigurationSources) (PromptKitProfileSourceConfig, promptkit.ConfigurationSources, []promptkit.Option, error) { profileSource, profileSources, err := promptKitProfileConfigurationSources(cfg) if err != nil { return PromptKitProfileSourceConfig{}, promptkit.ConfigurationSources{}, nil, err @@ -127,7 +128,9 @@ func promptKitEngineConfiguration(cfg PromptKitProfileSourceConfig, assets *Asse sources = maintainedSources options = append(options, maintainedOptions...) } + sources.Underlays = append(sources.Underlays, configuredSources.Underlays...) sources.Underlays = append(sources.Underlays, profileSources.Underlays...) + sources.Main = configuredSources.Main return profileSource, sources, options, nil } diff --git a/internal/framework/llm/promptkit_test_helpers_test.go b/internal/framework/llm/promptkit_test_helpers_test.go new file mode 100644 index 00000000..2b697f37 --- /dev/null +++ b/internal/framework/llm/promptkit_test_helpers_test.go @@ -0,0 +1,43 @@ +package llm + +import ( + "testing" + "testing/fstest" + + configkit "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +func promptKitTestString(value string) *string { return &value } + +func promptKitTestInt(value int) *int { return &value } + +func promptKitTestFloat(value float64) *float64 { return &value } + +func promptKitTestInputs(t *testing.T, label string, files fstest.MapFS, root string) []configkit.Input { + t.Helper() + inputs, err := promptkit.FilesystemInputs(label, files, root) + if err != nil { + t.Fatalf("FilesystemInputs() error = %v", err) + } + return inputs +} + +func promptKitTestSources( + backends map[string]promptkit.BackendConfig, + profiles map[string]promptkit.ProfileConfig, + otherInputs ...configkit.Input, +) promptkit.ConfigurationSources { + inputs := make([]configkit.Input, 0, len(otherInputs)+2) + if len(backends) != 0 { + inputs = append(inputs, configkit.Input{Source: promptkit.BackendSource("test backends", backends)}) + } + if len(profiles) != 0 { + inputs = append(inputs, configkit.Input{Source: promptkit.ProfileSource("test profiles", profiles)}) + } + inputs = append(inputs, otherInputs...) + if len(inputs) == 0 { + return promptkit.ConfigurationSources{} + } + return promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}} +}