Declare embedded prompts profiles and schemas with native kit resources
This commit is contained in:
159
assets/package_test.go
Normal file
159
assets/package_test.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package assets_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
type assetSettings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
|
||||
type noGeneration struct{ t *testing.T }
|
||||
|
||||
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.t.Error("resource preparation unexpectedly invoked generation")
|
||||
return nil, fmt.Errorf("unexpected generation")
|
||||
}
|
||||
|
||||
func loadNativeAssets(t *testing.T) (*configkit.Result[assetSettings], *promptkit.Engine) {
|
||||
t.Helper()
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
definition := promptkit.Configure(configkit.Definition[assetSettings]{}, func(value *assetSettings) *promptkit.Settings { return &value.PromptKit })
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[assetSettings]{
|
||||
Definition: definition,
|
||||
Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{
|
||||
Source: filesystem.File("notarius.yml"), Imports: true,
|
||||
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."},
|
||||
}}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
var loadError *configkit.LoadError
|
||||
if errors.As(err, &loadError) {
|
||||
for _, diagnostic := range loadError.Diagnostics() {
|
||||
t.Logf("%+v", diagnostic)
|
||||
}
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(noGeneration{t}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return result, engine
|
||||
}
|
||||
|
||||
func TestEmbeddedPromptsPrepareWithNativeContracts(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, engine := loadNativeAssets(t)
|
||||
// Discover the actual declarations as well as the composed inventory, so an
|
||||
// accidentally omitted import cannot silently remove a module's coverage.
|
||||
err := fs.WalkDir(assets.FS(), ".", func(file string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() || path.Base(file) != "prompt.yaml" {
|
||||
return nil
|
||||
}
|
||||
data, err := fs.ReadFile(assets.FS(), file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var declared struct {
|
||||
Prompts []promptkit.PromptSettings `yaml:"prompts"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &declared); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(declared.Prompts) == 0 {
|
||||
return fmt.Errorf("%s has no native prompts", file)
|
||||
}
|
||||
for _, prompt := range declared.Prompts {
|
||||
t.Run(prompt.ID, func(t *testing.T) {
|
||||
inputs := make(map[string]promptkit.ArtifactRef)
|
||||
for _, input := range prompt.Inputs {
|
||||
inputs[input.Name] = promptkit.Inline(input.Name + "-fixture")
|
||||
}
|
||||
request := promptkit.RunRequest{PromptID: "notarius/" + prompt.ID, Inputs: inputs, APIKey: "fixture-key"}
|
||||
if prompt.DefaultProfile == nil {
|
||||
request.ProfileID = "notarius/dnd-extraction"
|
||||
}
|
||||
prepared, err := engine.PrepareExecution(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Discard()
|
||||
details := prepared.Details()
|
||||
if details.OutputContract.Format != promptkit.FormatJSON || details.OutputContract.ValidationMode != promptkit.ValidationJSONSchema || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("missing native schema contract: %#v", details.OutputContract)
|
||||
}
|
||||
// Preparation must load the real self-contained schema, not merely retain
|
||||
// a filename or use a separate application schema registry.
|
||||
schemaBytes, err := fs.ReadFile(assets.FS(), details.OutputContract.SchemaPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var expected any
|
||||
if err := json.Unmarshal(schemaBytes, &expected); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
actualBytes, err := json.Marshal(details.StructuredOutput.JSONSchema.Schema)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var actual any
|
||||
if err := json.Unmarshal(actualBytes, &actual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(actual, expected) {
|
||||
t.Fatal("prepared schema differs from the owned asset")
|
||||
}
|
||||
first := prompt.Messages[0]
|
||||
if first.ContentFile == nil {
|
||||
t.Fatal("expected a source-relative initial message")
|
||||
}
|
||||
content, err := fs.ReadFile(assets.FS(), path.Join(path.Dir(file), *first.ContentFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(details.Messages) == 0 || details.Messages[0].Role != "system" || strings.TrimSpace(details.Messages[0].Content) != strings.TrimSpace(string(content)) {
|
||||
t.Fatal("shared initial system context was not preserved")
|
||||
}
|
||||
})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedProfileRetainsSelectedExecutionDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, engine := loadNativeAssets(t)
|
||||
profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if profile.Backend == nil || *profile.Backend != "openrouter" ||
|
||||
profile.Model == nil || *profile.Model != "openai/gpt-5.6-luna" ||
|
||||
profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 240 ||
|
||||
profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil {
|
||||
t.Fatalf("changed extraction defaults: %#v", profile)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user