Cover application-owned configuration handoff
This commit is contained in:
325
application_handoff_integration_test.go
Normal file
325
application_handoff_integration_test.go
Normal file
@@ -0,0 +1,325 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/configkit"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type integrationComponent struct {
|
||||||
|
ID string `yaml:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type integrationApplication struct {
|
||||||
|
Version string `yaml:"version"`
|
||||||
|
Sibling string `yaml:"sibling"`
|
||||||
|
Components []integrationComponent `yaml:"components"`
|
||||||
|
Prompts []string `yaml:"prompts"`
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func integrationDefinition(resolves, validates *atomic.Int64) configkit.Definition[integrationApplication] {
|
||||||
|
definition := configkit.Definition[integrationApplication]{
|
||||||
|
Defaults: func() integrationApplication { return integrationApplication{Sibling: "default"} },
|
||||||
|
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(version configkit.Version, validation *configkit.Validation) error {
|
||||||
|
if version.Present && version.Text != "app-v1" {
|
||||||
|
validation.Error(configkit.Keys("version"), "invalid application version")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}},
|
||||||
|
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("components"), "id")},
|
||||||
|
Sensitive: func(path configkit.Path) bool { return path.String() == "/sibling" },
|
||||||
|
Resolve: func(_ *integrationApplication, _ *configkit.Resolution) error { resolves.Add(1); return nil },
|
||||||
|
Validate: func(_ *integrationApplication, validation *configkit.Validation) {
|
||||||
|
validates.Add(1)
|
||||||
|
validation.Warning(configkit.Keys("sibling"), "application warning")
|
||||||
|
},
|
||||||
|
}
|
||||||
|
configured := promptkit.Configure(definition, func(value *integrationApplication) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
previous := configured.Validate
|
||||||
|
configured.Validate = func(value *integrationApplication, validation *configkit.Validation) {
|
||||||
|
previous(value, validation)
|
||||||
|
if len(value.PromptKit.Backends) != 1 || value.PromptKit.Backends[0].Endpoint == nil || strings.Contains(*value.PromptKit.Backends[0].Endpoint, " ") {
|
||||||
|
validation.Error(configkit.Keys("promptkit"), "backend normalization missing")
|
||||||
|
}
|
||||||
|
if value.PromptKit.Application["section_extra"] != "section" || value.PromptKit.Backends[0].Application["backend_extra"] != "backend" || value.PromptKit.Profiles[0].Application["profile_extra"] != "profile" || value.PromptKit.Prompts[0].Application["prompt_extra"] != "prompt" || value.PromptKit.Collections["group"].Application["collection_extra"] != "collection" {
|
||||||
|
validation.Error(configkit.Keys("promptkit"), "application fields missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return configured
|
||||||
|
}
|
||||||
|
|
||||||
|
const integrationDocument = `version: app-v1
|
||||||
|
prompts: [application-only]
|
||||||
|
components: [{id: second-component}]
|
||||||
|
promptkit:
|
||||||
|
section_extra: section
|
||||||
|
backends:
|
||||||
|
- id: local
|
||||||
|
endpoint: ' https://example.test/v1 '
|
||||||
|
extra_params: {provider: {mode: original}}
|
||||||
|
backend_extra: backend
|
||||||
|
profiles:
|
||||||
|
- id: profile
|
||||||
|
backend: local
|
||||||
|
model: original-model
|
||||||
|
profile_extra: profile
|
||||||
|
prompts:
|
||||||
|
- id: example
|
||||||
|
default_profile: profile
|
||||||
|
inputs: [{name: document, required: false}]
|
||||||
|
messages: [{role: user, content: hello}]
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema: {type: object, required: [ok], properties: {ok: {type: boolean}}}
|
||||||
|
prompt_extra: prompt
|
||||||
|
collections:
|
||||||
|
group: {collection_extra: collection}
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestApplicationOwnedLoadAndCapture(t *testing.T) {
|
||||||
|
var resolves, validates atomic.Int64
|
||||||
|
definition := integrationDefinition(&resolves, &validates)
|
||||||
|
result, err := configkit.Load(configkit.Bytes("application", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Config.Sibling != "default" || result.Config.Prompts[0] != "application-only" || len(result.Diagnostics()) != 1 || result.Config.PromptKit.Prompts[0].Output.RepairAttempts == nil || *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("loaded application = %+v, diagnostics = %+v", result.Config, result.Diagnostics())
|
||||||
|
}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"ok":true}`}}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if resolves.Load() != 1 || validates.Load() != 1 {
|
||||||
|
t.Fatalf("callbacks rerun during handoff: %d %d", resolves.Load(), validates.Load())
|
||||||
|
}
|
||||||
|
before, err := engine.InspectPrompt(context.Background(), "example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
app := &result.Config.PromptKit
|
||||||
|
*app.Backends[0].Endpoint = "https://changed.test/v1"
|
||||||
|
app.Backends[0].ExtraParams["provider"].(map[string]any)["mode"] = "changed"
|
||||||
|
*app.Profiles[0].Model = "changed-model"
|
||||||
|
*app.Prompts[0].Messages[0].Content = "changed"
|
||||||
|
app.Prompts[0].Inputs[0].Name = "changed"
|
||||||
|
app.Prompts[0].Output.Schema.(map[string]any)["required"].([]any)[0] = "changed"
|
||||||
|
app.Prompts = nil
|
||||||
|
app.Collections = nil
|
||||||
|
after, err := engine.InspectPrompt(context.Background(), "example")
|
||||||
|
if err != nil || !reflect.DeepEqual(before, after) {
|
||||||
|
t.Fatalf("captured prompt changed: %+v, %+v, %v", before, after, err)
|
||||||
|
}
|
||||||
|
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "example"})
|
||||||
|
if err != nil || run.Validation.Status != promptkit.ValidationPassed || len(client.requests) != 1 || client.requests[0].Prompt.Messages[0].Content != "hello" || client.requests[0].Target.Model != "original-model" {
|
||||||
|
t.Fatalf("run = %+v, requests = %+v, error = %v", run, client.requests, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(fmt.Sprintf("%+v", client.requests[0].Target.ExtraParams), "backend_extra") || strings.Contains(fmt.Sprintf("%+v", after), "prompt_extra") {
|
||||||
|
t.Fatal("application fields leaked into resources or provider parameters")
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var report bytes.Buffer
|
||||||
|
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "section_extra: section") || !strings.Contains(report.String(), "prompt_extra: prompt") {
|
||||||
|
t.Fatalf("inspection = %q, %v", report.String(), err)
|
||||||
|
}
|
||||||
|
if len(view.Diagnostics()) != 1 {
|
||||||
|
t.Fatalf("diagnostics = %+v", view.Diagnostics())
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range 4 {
|
||||||
|
wg.Go(func() {
|
||||||
|
_, err := configkit.Load(configkit.Bytes("repeat", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if resolves.Load() != 5 || validates.Load() != 5 {
|
||||||
|
t.Fatalf("reused callbacks = %d %d", resolves.Load(), validates.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandoffMountedSourcesAndExecution(t *testing.T) {
|
||||||
|
baseFS := &countingMountFS{FS: fstest.MapFS{
|
||||||
|
"base/collection.yaml": {Data: []byte("prompts:\n - id: parent\n default_profile: profile\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")},
|
||||||
|
"base/original.txt": {Data: []byte("original")}, "base/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||||
|
}}
|
||||||
|
operatorFS := &countingMountFS{FS: fstest.MapFS{
|
||||||
|
"operator/item.yaml": {Data: []byte("id: child\nextends: {id: parent}\nmessages: [{role: user, content_file: replacement.txt}]\n")},
|
||||||
|
"operator/replacement.txt": {Data: []byte("replacement")},
|
||||||
|
}}
|
||||||
|
base := configkit.NewFilesystem("embedded", baseFS)
|
||||||
|
operator := configkit.NewFilesystem("operator", operatorFS)
|
||||||
|
collection, err := promptkit.CollectionInput("team", base.File("base/collection.yaml"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
collection.Filesystem = &configkit.FilesystemContext{FS: &base, Root: "."}
|
||||||
|
item := configkit.Input{Source: operator.File("operator/item.yaml"), Destination: configkit.Keys("promptkit", "collections", "team", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem, Filesystem: &configkit.FilesystemContext{FS: &operator, Root: "."}}
|
||||||
|
profile := promptkit.SettingsInput(configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.test/v1", "model": "model"}}}))
|
||||||
|
type root struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
layers := []configkit.Underlay{{Inputs: []configkit.Input{profile, collection}}, {Inputs: []configkit.Input{item}}}
|
||||||
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: layers})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
beforeBase, beforeOperator := baseFS.opens, operatorFS.opens
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{}`}}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if baseFS.opens-beforeBase != 1 || operatorFS.opens-beforeOperator != 1 {
|
||||||
|
t.Fatalf("unexpected construction reads: base=%d operator=%d", baseFS.opens-beforeBase, operatorFS.opens-beforeOperator)
|
||||||
|
}
|
||||||
|
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: layers}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantPrompt, err := standalone.InspectPrompt(context.Background(), "team/child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gotPrompt, err := engine.InspectPrompt(context.Background(), "team/child")
|
||||||
|
if err != nil || !reflect.DeepEqual(gotPrompt, wantPrompt) {
|
||||||
|
t.Fatalf("resource reports = %+v, %+v, %v", gotPrompt, wantPrompt, err)
|
||||||
|
}
|
||||||
|
wantInventory, err := standalone.ListPrompts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gotInventory, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || !reflect.DeepEqual(gotInventory, wantInventory) {
|
||||||
|
t.Fatalf("inventories = %+v, %+v, %v", gotInventory, wantInventory, err)
|
||||||
|
}
|
||||||
|
collectionSettings := result.Config.PromptKit.Collections["team"]
|
||||||
|
*collectionSettings.Prompts[1].Output.SchemaFile = "changed.json"
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||||
|
if err != nil || prepared.Messages[0].Content != "replacement" {
|
||||||
|
t.Fatalf("prepared = %+v, %v", prepared, err)
|
||||||
|
}
|
||||||
|
standalonePrepared, err := standalone.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||||
|
if err != nil || !reflect.DeepEqual(prepared.StructuredOutput, standalonePrepared.StructuredOutput) || !reflect.DeepEqual(prepared.OutputContract, standalonePrepared.OutputContract) || !reflect.DeepEqual(prepared.Messages, standalonePrepared.Messages) {
|
||||||
|
t.Fatalf("prepared details differ: %+v, %+v, %v", prepared, standalonePrepared, err)
|
||||||
|
}
|
||||||
|
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||||
|
if err != nil || run.Validation.Status != promptkit.ValidationPassed {
|
||||||
|
t.Fatalf("run = %+v, %v", run, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandoffUsesOnlyBundlesSelectedBeforeLoad(t *testing.T) {
|
||||||
|
type root struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{promptkit.BundledOpenRouterUnderlay()}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
backends, err := engine.ListBackends(context.Background())
|
||||||
|
if err != nil || len(backends) != 1 || backends[0] != "openrouter" {
|
||||||
|
t.Fatalf("bundles = %+v, %v", backends, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationFieldsComposeAndInherit(t *testing.T) {
|
||||||
|
type root struct {
|
||||||
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||||
|
underlay := promptkit.SettingsInput(configkit.Mapping("defaults", map[string]any{
|
||||||
|
"section_data": map[string]any{"first": "section"},
|
||||||
|
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.test/v1", "app_data": map[string]any{"first": "backend"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "base", "model": "model", "app_data": map[string]any{"first": "profile"}}},
|
||||||
|
"prompts": []any{map[string]any{"id": "base", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "app_data": map[string]any{"first": "prompt"}}},
|
||||||
|
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"first": "collection"}}},
|
||||||
|
}))
|
||||||
|
main := promptkit.SettingsInput(configkit.Mapping("operator", map[string]any{
|
||||||
|
"section_data": map[string]any{"second": "section"},
|
||||||
|
"backends": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "backend"}}},
|
||||||
|
"profiles": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "profile"}}},
|
||||||
|
"prompts": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "prompt"}}},
|
||||||
|
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"second": "collection"}}},
|
||||||
|
}))
|
||||||
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
settings := result.Config.PromptKit
|
||||||
|
for name, fields := range map[string]map[string]any{"section": settings.Application, "backend": settings.Backends[1].Application, "profile": settings.Profiles[1].Application, "prompt": settings.Prompts[1].Application, "collection": settings.Collections["team"].Application} {
|
||||||
|
key := "app_data"
|
||||||
|
if name == "section" {
|
||||||
|
key = "section_data"
|
||||||
|
}
|
||||||
|
data, ok := fields[key].(map[string]any)
|
||||||
|
if !ok || data["first"] != name || data["second"] != name {
|
||||||
|
t.Errorf("%s extension = %#v", name, fields[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandoffMissingAndNullSections(t *testing.T) {
|
||||||
|
type noSection struct {
|
||||||
|
Prompts []string `yaml:"prompts"`
|
||||||
|
}
|
||||||
|
missing, err := configkit.Load[noSection](configkit.Bytes("application-only", []byte("prompts: [application]\n")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
empty, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if prompts, err := empty.ListPrompts(context.Background()); err != nil || len(prompts) != 0 {
|
||||||
|
t.Fatalf("application root resources were consumed: %+v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
for _, source := range []string{"{}", "promptkit: null", "promptkit: {}"} {
|
||||||
|
t.Run(source, func(t *testing.T) {
|
||||||
|
type root struct {
|
||||||
|
PromptKit *promptkit.Settings `yaml:"promptkit"`
|
||||||
|
Prompts []string `yaml:"prompts"`
|
||||||
|
}
|
||||||
|
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return value.PromptKit })
|
||||||
|
result, err := configkit.Load(configkit.Bytes("missing", []byte(source)), configkit.Options[root]{Definition: definition})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompts, err := engine.ListPrompts(context.Background())
|
||||||
|
if err != nil || len(prompts) != 0 {
|
||||||
|
t.Fatalf("prompts = %+v, %v", prompts, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := promptkit.NewFromConfig(nil, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil result = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -553,6 +553,9 @@ and configuration contracts; expected test call-site migrations are complete.
|
|||||||
|
|
||||||
## Stage 13 — Add Application-Owned Handoff Integration Coverage
|
## Stage 13 — Add Application-Owned Handoff Integration Coverage
|
||||||
|
|
||||||
|
Status: complete. External handoff integration tests passed under the race
|
||||||
|
detector, including concurrent loads, source contexts, and ownership checks.
|
||||||
|
|
||||||
Focus: a small external-package root integration test group, with real ConfigKit
|
Focus: a small external-package root integration test group, with real ConfigKit
|
||||||
and PromptKit collaborators and an offline model client.
|
and PromptKit collaborators and an offline model client.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user