package promptkit_test import ( "bytes" "context" "errors" "fmt" "reflect" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) type integrationComponent struct { ID string `yaml:"id"` } type integrationApplication struct { Version string `yaml:"version"` Sibling string `yaml:"sibling"` Components []integrationComponent `yaml:"components"` Prompts []string `yaml:"prompts"` PromptKit promptkit.Settings `yaml:"promptkit"` } func integrationDefinition(resolves, validates *atomic.Int64) configkit.Definition[integrationApplication] { definition := configkit.Definition[integrationApplication]{ Defaults: func() integrationApplication { return integrationApplication{Sibling: "default"} }, Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(version configkit.Version, validation *configkit.Validation) error { if version.Present && version.Text != "app-v1" { validation.Error(configkit.Keys("version"), "invalid application version") } return nil }}, SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("components"), "id")}, Sensitive: func(path configkit.Path) bool { return path.String() == "/sibling" }, Resolve: func(_ *integrationApplication, _ *configkit.Resolution) error { resolves.Add(1); return nil }, Validate: func(_ *integrationApplication, validation *configkit.Validation) { validates.Add(1) validation.Warning(configkit.Keys("sibling"), "application warning") }, } configured := promptkit.Configure(definition, func(value *integrationApplication) *promptkit.Settings { return &value.PromptKit }) previous := configured.Validate configured.Validate = func(value *integrationApplication, validation *configkit.Validation) { previous(value, validation) if len(value.PromptKit.Backends) != 1 || value.PromptKit.Backends[0].Endpoint == nil || strings.Contains(*value.PromptKit.Backends[0].Endpoint, " ") { validation.Error(configkit.Keys("promptkit"), "backend normalization missing") } if value.PromptKit.Application["section_extra"] != "section" || value.PromptKit.Backends[0].Application["backend_extra"] != "backend" || value.PromptKit.Profiles[0].Application["profile_extra"] != "profile" || value.PromptKit.Prompts[0].Application["prompt_extra"] != "prompt" || value.PromptKit.Collections["group"].Application["collection_extra"] != "collection" { validation.Error(configkit.Keys("promptkit"), "application fields missing") } } return configured } const integrationDocument = `version: app-v1 prompts: [application-only] components: [{id: second-component}] promptkit: section_extra: section backends: - id: local endpoint: ' https://example.test/v1 ' extra_params: {provider: {mode: original}} backend_extra: backend profiles: - id: profile backend: local model: original-model profile_extra: profile prompts: - id: example default_profile: profile inputs: [{name: document, required: false}] messages: [{role: user, content: hello}] output: format: json validation_mode: json_schema schema: {type: object, required: [ok], properties: {ok: {type: boolean}}} prompt_extra: prompt collections: group: {collection_extra: collection} ` func TestApplicationOwnedLoadAndCapture(t *testing.T) { var resolves, validates atomic.Int64 definition := integrationDefinition(&resolves, &validates) result, err := configkit.Load(configkit.Bytes("application", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition}) if err != nil { t.Fatal(err) } if result.Config.Sibling != "default" || result.Config.Prompts[0] != "application-only" || len(result.Diagnostics()) != 1 || result.Config.PromptKit.Prompts[0].Output.RepairAttempts == nil || *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 { t.Fatalf("loaded application = %+v, diagnostics = %+v", result.Config, result.Diagnostics()) } client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"ok":true}`}} engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) } if resolves.Load() != 1 || validates.Load() != 1 { t.Fatalf("callbacks rerun during handoff: %d %d", resolves.Load(), validates.Load()) } before, err := engine.InspectPrompt(context.Background(), "example") if err != nil { t.Fatal(err) } app := &result.Config.PromptKit *app.Backends[0].Endpoint = "https://changed.test/v1" app.Backends[0].ExtraParams["provider"].(map[string]any)["mode"] = "changed" *app.Profiles[0].Model = "changed-model" *app.Prompts[0].Messages[0].Content = "changed" app.Prompts[0].Inputs[0].Name = "changed" app.Prompts[0].Output.Schema.(map[string]any)["required"].([]any)[0] = "changed" app.Prompts = nil app.Collections = nil after, err := engine.InspectPrompt(context.Background(), "example") if err != nil || !reflect.DeepEqual(before, after) { t.Fatalf("captured prompt changed: %+v, %+v, %v", before, after, err) } run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "example"}) if err != nil || run.Validation.Status != promptkit.ValidationPassed || len(client.requests) != 1 || client.requests[0].Prompt.Messages[0].Content != "hello" || client.requests[0].Target.Model != "original-model" { t.Fatalf("run = %+v, requests = %+v, error = %v", run, client.requests, err) } if strings.Contains(fmt.Sprintf("%+v", client.requests[0].Target.ExtraParams), "backend_extra") || strings.Contains(fmt.Sprintf("%+v", after), "prompt_extra") { t.Fatal("application fields leaked into resources or provider parameters") } view, err := engine.InspectConfiguration() if err != nil { t.Fatal(err) } var report bytes.Buffer if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "section_extra: section") || !strings.Contains(report.String(), "prompt_extra: prompt") { t.Fatalf("inspection = %q, %v", report.String(), err) } if len(view.Diagnostics()) != 1 { t.Fatalf("diagnostics = %+v", view.Diagnostics()) } var wg sync.WaitGroup for range 4 { wg.Go(func() { _, err := configkit.Load(configkit.Bytes("repeat", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition}) if err != nil { t.Error(err) } }) } wg.Wait() if resolves.Load() != 5 || validates.Load() != 5 { t.Fatalf("reused callbacks = %d %d", resolves.Load(), validates.Load()) } } func TestHandoffMountedSourcesAndExecution(t *testing.T) { baseFS := &countingMountFS{FS: fstest.MapFS{ "base/collection.yaml": {Data: []byte("prompts:\n - id: parent\n default_profile: profile\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")}, "base/original.txt": {Data: []byte("original")}, "base/schema.json": {Data: []byte(`{"type":"object"}`)}, }} operatorFS := &countingMountFS{FS: fstest.MapFS{ "operator/item.yaml": {Data: []byte("id: child\nextends: {id: parent}\nmessages: [{role: user, content_file: replacement.txt}]\n")}, "operator/replacement.txt": {Data: []byte("replacement")}, }} base := configkit.NewFilesystem("embedded", baseFS) operator := configkit.NewFilesystem("operator", operatorFS) collection, err := promptkit.CollectionInput("team", base.File("base/collection.yaml")) if err != nil { t.Fatal(err) } collection.Filesystem = &configkit.FilesystemContext{FS: &base, Root: "."} item := configkit.Input{Source: operator.File("operator/item.yaml"), Destination: configkit.Keys("promptkit", "collections", "team", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem, Filesystem: &configkit.FilesystemContext{FS: &operator, Root: "."}} profile := promptkit.SettingsInput(configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.test/v1", "model": "model"}}})) type root struct { PromptKit promptkit.Settings `yaml:"promptkit"` } definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit }) layers := []configkit.Underlay{{Inputs: []configkit.Input{profile, collection}}, {Inputs: []configkit.Input{item}}} result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: layers}) if err != nil { t.Fatal(err) } beforeBase, beforeOperator := baseFS.opens, operatorFS.opens client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{}`}} engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) } if baseFS.opens-beforeBase != 1 || operatorFS.opens-beforeOperator != 1 { t.Fatalf("unexpected construction reads: base=%d operator=%d", baseFS.opens-beforeBase, operatorFS.opens-beforeOperator) } standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: layers}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) } wantPrompt, err := standalone.InspectPrompt(context.Background(), "team/child") if err != nil { t.Fatal(err) } gotPrompt, err := engine.InspectPrompt(context.Background(), "team/child") if err != nil || !reflect.DeepEqual(gotPrompt, wantPrompt) { t.Fatalf("resource reports = %+v, %+v, %v", gotPrompt, wantPrompt, err) } wantInventory, err := standalone.ListPrompts(context.Background()) if err != nil { t.Fatal(err) } gotInventory, err := engine.ListPrompts(context.Background()) if err != nil || !reflect.DeepEqual(gotInventory, wantInventory) { t.Fatalf("inventories = %+v, %+v, %v", gotInventory, wantInventory, err) } collectionSettings := result.Config.PromptKit.Collections["team"] *collectionSettings.Prompts[1].Output.SchemaFile = "changed.json" prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"}) if err != nil || prepared.Messages[0].Content != "replacement" { t.Fatalf("prepared = %+v, %v", prepared, err) } standalonePrepared, err := standalone.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"}) if err != nil || !reflect.DeepEqual(prepared.StructuredOutput, standalonePrepared.StructuredOutput) || !reflect.DeepEqual(prepared.OutputContract, standalonePrepared.OutputContract) || !reflect.DeepEqual(prepared.Messages, standalonePrepared.Messages) { t.Fatalf("prepared details differ: %+v, %+v, %v", prepared, standalonePrepared, err) } run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "team/child"}) if err != nil || run.Validation.Status != promptkit.ValidationPassed { t.Fatalf("run = %+v, %v", run, err) } } func TestHandoffUsesOnlyBundlesSelectedBeforeLoad(t *testing.T) { type root struct { PromptKit promptkit.Settings `yaml:"promptkit"` } definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit }) result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{promptkit.BundledOpenRouterUnderlay()}}) if err != nil { t.Fatal(err) } engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(false)) if err != nil { t.Fatal(err) } backends, err := engine.ListBackends(context.Background()) if err != nil || len(backends) != 1 || backends[0] != "openrouter" { t.Fatalf("bundles = %+v, %v", backends, err) } } func TestApplicationFieldsComposeAndInherit(t *testing.T) { type root struct { PromptKit promptkit.Settings `yaml:"promptkit"` } definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit }) underlay := promptkit.SettingsInput(configkit.Mapping("defaults", map[string]any{ "section_data": map[string]any{"first": "section"}, "backends": []any{map[string]any{"id": "base", "endpoint": "https://example.test/v1", "app_data": map[string]any{"first": "backend"}}}, "profiles": []any{map[string]any{"id": "base", "model": "model", "app_data": map[string]any{"first": "profile"}}}, "prompts": []any{map[string]any{"id": "base", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "app_data": map[string]any{"first": "prompt"}}}, "collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"first": "collection"}}}, })) main := promptkit.SettingsInput(configkit.Mapping("operator", map[string]any{ "section_data": map[string]any{"second": "section"}, "backends": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "backend"}}}, "profiles": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "profile"}}}, "prompts": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "prompt"}}}, "collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"second": "collection"}}}, })) result, err := configkit.LoadLayered(main, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}}) if err != nil { t.Fatal(err) } settings := result.Config.PromptKit for name, fields := range map[string]map[string]any{"section": settings.Application, "backend": settings.Backends[1].Application, "profile": settings.Profiles[1].Application, "prompt": settings.Prompts[1].Application, "collection": settings.Collections["team"].Application} { key := "app_data" if name == "section" { key = "section_data" } data, ok := fields[key].(map[string]any) if !ok || data["first"] != name || data["second"] != name { t.Errorf("%s extension = %#v", name, fields[key]) } } } func TestHandoffMissingAndNullSections(t *testing.T) { type noSection struct { Prompts []string `yaml:"prompts"` } missing, err := configkit.Load[noSection](configkit.Bytes("application-only", []byte("prompts: [application]\n"))) if err != nil { t.Fatal(err) } empty, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{}) if err != nil { t.Fatal(err) } if prompts, err := empty.ListPrompts(context.Background()); err != nil || len(prompts) != 0 { t.Fatalf("application root resources were consumed: %+v, %v", prompts, err) } for _, source := range []string{"{}", "promptkit: null", "promptkit: {}"} { t.Run(source, func(t *testing.T) { type root struct { PromptKit *promptkit.Settings `yaml:"promptkit"` Prompts []string `yaml:"prompts"` } definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return value.PromptKit }) result, err := configkit.Load(configkit.Bytes("missing", []byte(source)), configkit.Options[root]{Definition: definition}) if err != nil { t.Fatal(err) } engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}) if err != nil { t.Fatal(err) } prompts, err := engine.ListPrompts(context.Background()) if err != nil || len(prompts) != 0 { t.Fatalf("prompts = %+v, %v", prompts, err) } }) } if _, err := promptkit.NewFromConfig(nil, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("nil result = %v", err) } }