Cover application-owned configuration handoff
This commit is contained in:
325
application_handoff_integration_test.go
Normal file
325
application_handoff_integration_test.go
Normal file
@@ -0,0 +1,325 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type integrationComponent struct {
|
||||
ID string `yaml:"id"`
|
||||
}
|
||||
|
||||
type integrationApplication struct {
|
||||
Version string `yaml:"version"`
|
||||
Sibling string `yaml:"sibling"`
|
||||
Components []integrationComponent `yaml:"components"`
|
||||
Prompts []string `yaml:"prompts"`
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
func integrationDefinition(resolves, validates *atomic.Int64) configkit.Definition[integrationApplication] {
|
||||
definition := configkit.Definition[integrationApplication]{
|
||||
Defaults: func() integrationApplication { return integrationApplication{Sibling: "default"} },
|
||||
Version: &configkit.VersionRule{Path: configkit.Keys("version"), Check: func(version configkit.Version, validation *configkit.Validation) error {
|
||||
if version.Present && version.Text != "app-v1" {
|
||||
validation.Error(configkit.Keys("version"), "invalid application version")
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
SequenceIdentities: []configkit.SequenceIdentity{configkit.SequenceIdentityAt(configkit.Keys("components"), "id")},
|
||||
Sensitive: func(path configkit.Path) bool { return path.String() == "/sibling" },
|
||||
Resolve: func(_ *integrationApplication, _ *configkit.Resolution) error { resolves.Add(1); return nil },
|
||||
Validate: func(_ *integrationApplication, validation *configkit.Validation) {
|
||||
validates.Add(1)
|
||||
validation.Warning(configkit.Keys("sibling"), "application warning")
|
||||
},
|
||||
}
|
||||
configured := promptkit.Configure(definition, func(value *integrationApplication) *promptkit.Settings { return &value.PromptKit })
|
||||
previous := configured.Validate
|
||||
configured.Validate = func(value *integrationApplication, validation *configkit.Validation) {
|
||||
previous(value, validation)
|
||||
if len(value.PromptKit.Backends) != 1 || value.PromptKit.Backends[0].Endpoint == nil || strings.Contains(*value.PromptKit.Backends[0].Endpoint, " ") {
|
||||
validation.Error(configkit.Keys("promptkit"), "backend normalization missing")
|
||||
}
|
||||
if value.PromptKit.Application["section_extra"] != "section" || value.PromptKit.Backends[0].Application["backend_extra"] != "backend" || value.PromptKit.Profiles[0].Application["profile_extra"] != "profile" || value.PromptKit.Prompts[0].Application["prompt_extra"] != "prompt" || value.PromptKit.Collections["group"].Application["collection_extra"] != "collection" {
|
||||
validation.Error(configkit.Keys("promptkit"), "application fields missing")
|
||||
}
|
||||
}
|
||||
return configured
|
||||
}
|
||||
|
||||
const integrationDocument = `version: app-v1
|
||||
prompts: [application-only]
|
||||
components: [{id: second-component}]
|
||||
promptkit:
|
||||
section_extra: section
|
||||
backends:
|
||||
- id: local
|
||||
endpoint: ' https://example.test/v1 '
|
||||
extra_params: {provider: {mode: original}}
|
||||
backend_extra: backend
|
||||
profiles:
|
||||
- id: profile
|
||||
backend: local
|
||||
model: original-model
|
||||
profile_extra: profile
|
||||
prompts:
|
||||
- id: example
|
||||
default_profile: profile
|
||||
inputs: [{name: document, required: false}]
|
||||
messages: [{role: user, content: hello}]
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema: {type: object, required: [ok], properties: {ok: {type: boolean}}}
|
||||
prompt_extra: prompt
|
||||
collections:
|
||||
group: {collection_extra: collection}
|
||||
`
|
||||
|
||||
func TestApplicationOwnedLoadAndCapture(t *testing.T) {
|
||||
var resolves, validates atomic.Int64
|
||||
definition := integrationDefinition(&resolves, &validates)
|
||||
result, err := configkit.Load(configkit.Bytes("application", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Config.Sibling != "default" || result.Config.Prompts[0] != "application-only" || len(result.Diagnostics()) != 1 || result.Config.PromptKit.Prompts[0].Output.RepairAttempts == nil || *result.Config.PromptKit.Prompts[0].Output.RepairAttempts != 1 {
|
||||
t.Fatalf("loaded application = %+v, diagnostics = %+v", result.Config, result.Diagnostics())
|
||||
}
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"ok":true}`}}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolves.Load() != 1 || validates.Load() != 1 {
|
||||
t.Fatalf("callbacks rerun during handoff: %d %d", resolves.Load(), validates.Load())
|
||||
}
|
||||
before, err := engine.InspectPrompt(context.Background(), "example")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app := &result.Config.PromptKit
|
||||
*app.Backends[0].Endpoint = "https://changed.test/v1"
|
||||
app.Backends[0].ExtraParams["provider"].(map[string]any)["mode"] = "changed"
|
||||
*app.Profiles[0].Model = "changed-model"
|
||||
*app.Prompts[0].Messages[0].Content = "changed"
|
||||
app.Prompts[0].Inputs[0].Name = "changed"
|
||||
app.Prompts[0].Output.Schema.(map[string]any)["required"].([]any)[0] = "changed"
|
||||
app.Prompts = nil
|
||||
app.Collections = nil
|
||||
after, err := engine.InspectPrompt(context.Background(), "example")
|
||||
if err != nil || !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("captured prompt changed: %+v, %+v, %v", before, after, err)
|
||||
}
|
||||
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "example"})
|
||||
if err != nil || run.Validation.Status != promptkit.ValidationPassed || len(client.requests) != 1 || client.requests[0].Prompt.Messages[0].Content != "hello" || client.requests[0].Target.Model != "original-model" {
|
||||
t.Fatalf("run = %+v, requests = %+v, error = %v", run, client.requests, err)
|
||||
}
|
||||
if strings.Contains(fmt.Sprintf("%+v", client.requests[0].Target.ExtraParams), "backend_extra") || strings.Contains(fmt.Sprintf("%+v", after), "prompt_extra") {
|
||||
t.Fatal("application fields leaked into resources or provider parameters")
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var report bytes.Buffer
|
||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "section_extra: section") || !strings.Contains(report.String(), "prompt_extra: prompt") {
|
||||
t.Fatalf("inspection = %q, %v", report.String(), err)
|
||||
}
|
||||
if len(view.Diagnostics()) != 1 {
|
||||
t.Fatalf("diagnostics = %+v", view.Diagnostics())
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for range 4 {
|
||||
wg.Go(func() {
|
||||
_, err := configkit.Load(configkit.Bytes("repeat", []byte(integrationDocument)), configkit.Options[integrationApplication]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if resolves.Load() != 5 || validates.Load() != 5 {
|
||||
t.Fatalf("reused callbacks = %d %d", resolves.Load(), validates.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandoffMountedSourcesAndExecution(t *testing.T) {
|
||||
baseFS := &countingMountFS{FS: fstest.MapFS{
|
||||
"base/collection.yaml": {Data: []byte("prompts:\n - id: parent\n default_profile: profile\n messages: [{role: user, content_file: original.txt}]\n output: {format: json, validation_mode: json_schema, schema_file: schema.json}\n")},
|
||||
"base/original.txt": {Data: []byte("original")}, "base/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}}
|
||||
operatorFS := &countingMountFS{FS: fstest.MapFS{
|
||||
"operator/item.yaml": {Data: []byte("id: child\nextends: {id: parent}\nmessages: [{role: user, content_file: replacement.txt}]\n")},
|
||||
"operator/replacement.txt": {Data: []byte("replacement")},
|
||||
}}
|
||||
base := configkit.NewFilesystem("embedded", baseFS)
|
||||
operator := configkit.NewFilesystem("operator", operatorFS)
|
||||
collection, err := promptkit.CollectionInput("team", base.File("base/collection.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
collection.Filesystem = &configkit.FilesystemContext{FS: &base, Root: "."}
|
||||
item := configkit.Input{Source: operator.File("operator/item.yaml"), Destination: configkit.Keys("promptkit", "collections", "team", "prompts"), PlacementMode: configkit.PlacementModeSequenceItem, Filesystem: &configkit.FilesystemContext{FS: &operator, Root: "."}}
|
||||
profile := promptkit.SettingsInput(configkit.Mapping("profile", map[string]any{"profiles": []any{map[string]any{"id": "profile", "endpoint": "https://example.test/v1", "model": "model"}}}))
|
||||
type root struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||
layers := []configkit.Underlay{{Inputs: []configkit.Input{profile, collection}}, {Inputs: []configkit.Input{item}}}
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: layers})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
beforeBase, beforeOperator := baseFS.opens, operatorFS.opens
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{}`}}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if baseFS.opens-beforeBase != 1 || operatorFS.opens-beforeOperator != 1 {
|
||||
t.Fatalf("unexpected construction reads: base=%d operator=%d", baseFS.opens-beforeBase, operatorFS.opens-beforeOperator)
|
||||
}
|
||||
standalone, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: layers}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPrompt, err := standalone.InspectPrompt(context.Background(), "team/child")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotPrompt, err := engine.InspectPrompt(context.Background(), "team/child")
|
||||
if err != nil || !reflect.DeepEqual(gotPrompt, wantPrompt) {
|
||||
t.Fatalf("resource reports = %+v, %+v, %v", gotPrompt, wantPrompt, err)
|
||||
}
|
||||
wantInventory, err := standalone.ListPrompts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotInventory, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || !reflect.DeepEqual(gotInventory, wantInventory) {
|
||||
t.Fatalf("inventories = %+v, %+v, %v", gotInventory, wantInventory, err)
|
||||
}
|
||||
collectionSettings := result.Config.PromptKit.Collections["team"]
|
||||
*collectionSettings.Prompts[1].Output.SchemaFile = "changed.json"
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||
if err != nil || prepared.Messages[0].Content != "replacement" {
|
||||
t.Fatalf("prepared = %+v, %v", prepared, err)
|
||||
}
|
||||
standalonePrepared, err := standalone.Prepare(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||
if err != nil || !reflect.DeepEqual(prepared.StructuredOutput, standalonePrepared.StructuredOutput) || !reflect.DeepEqual(prepared.OutputContract, standalonePrepared.OutputContract) || !reflect.DeepEqual(prepared.Messages, standalonePrepared.Messages) {
|
||||
t.Fatalf("prepared details differ: %+v, %+v, %v", prepared, standalonePrepared, err)
|
||||
}
|
||||
run, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "team/child"})
|
||||
if err != nil || run.Validation.Status != promptkit.ValidationPassed {
|
||||
t.Fatalf("run = %+v, %v", run, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandoffUsesOnlyBundlesSelectedBeforeLoad(t *testing.T) {
|
||||
type root struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{promptkit.BundledOpenRouterUnderlay()}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithBundledOpenRouter(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
backends, err := engine.ListBackends(context.Background())
|
||||
if err != nil || len(backends) != 1 || backends[0] != "openrouter" {
|
||||
t.Fatalf("bundles = %+v, %v", backends, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationFieldsComposeAndInherit(t *testing.T) {
|
||||
type root struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return &value.PromptKit })
|
||||
underlay := promptkit.SettingsInput(configkit.Mapping("defaults", map[string]any{
|
||||
"section_data": map[string]any{"first": "section"},
|
||||
"backends": []any{map[string]any{"id": "base", "endpoint": "https://example.test/v1", "app_data": map[string]any{"first": "backend"}}},
|
||||
"profiles": []any{map[string]any{"id": "base", "model": "model", "app_data": map[string]any{"first": "profile"}}},
|
||||
"prompts": []any{map[string]any{"id": "base", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "none"}, "app_data": map[string]any{"first": "prompt"}}},
|
||||
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"first": "collection"}}},
|
||||
}))
|
||||
main := promptkit.SettingsInput(configkit.Mapping("operator", map[string]any{
|
||||
"section_data": map[string]any{"second": "section"},
|
||||
"backends": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "backend"}}},
|
||||
"profiles": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "profile"}}},
|
||||
"prompts": []any{map[string]any{"id": "child", "extends": map[string]any{"id": "base"}, "app_data": map[string]any{"second": "prompt"}}},
|
||||
"collections": map[string]any{"team": map[string]any{"app_data": map[string]any{"second": "collection"}}},
|
||||
}))
|
||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[root]{Definition: definition, Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := result.Config.PromptKit
|
||||
for name, fields := range map[string]map[string]any{"section": settings.Application, "backend": settings.Backends[1].Application, "profile": settings.Profiles[1].Application, "prompt": settings.Prompts[1].Application, "collection": settings.Collections["team"].Application} {
|
||||
key := "app_data"
|
||||
if name == "section" {
|
||||
key = "section_data"
|
||||
}
|
||||
data, ok := fields[key].(map[string]any)
|
||||
if !ok || data["first"] != name || data["second"] != name {
|
||||
t.Errorf("%s extension = %#v", name, fields[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandoffMissingAndNullSections(t *testing.T) {
|
||||
type noSection struct {
|
||||
Prompts []string `yaml:"prompts"`
|
||||
}
|
||||
missing, err := configkit.Load[noSection](configkit.Bytes("application-only", []byte("prompts: [application]\n")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
empty, err := promptkit.NewFromConfig(missing, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prompts, err := empty.ListPrompts(context.Background()); err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("application root resources were consumed: %+v, %v", prompts, err)
|
||||
}
|
||||
for _, source := range []string{"{}", "promptkit: null", "promptkit: {}"} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
type root struct {
|
||||
PromptKit *promptkit.Settings `yaml:"promptkit"`
|
||||
Prompts []string `yaml:"prompts"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[root]{}, func(value *root) *promptkit.Settings { return value.PromptKit })
|
||||
result, err := configkit.Load(configkit.Bytes("missing", []byte(source)), configkit.Options[root]{Definition: definition})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prompts, err := engine.ListPrompts(context.Background())
|
||||
if err != nil || len(prompts) != 0 {
|
||||
t.Fatalf("prompts = %+v, %v", prompts, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := promptkit.NewFromConfig(nil, promptkit.RuntimeConfig{}); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("nil result = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -553,6 +553,9 @@ and configuration contracts; expected test call-site migrations are complete.
|
||||
|
||||
## Stage 13 — Add Application-Owned Handoff Integration Coverage
|
||||
|
||||
Status: complete. External handoff integration tests passed under the race
|
||||
detector, including concurrent loads, source contexts, and ownership checks.
|
||||
|
||||
Focus: a small external-package root integration test group, with real ConfigKit
|
||||
and PromptKit collaborators and an offline model client.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user