163 lines
5.4 KiB
Go
163 lines
5.4 KiB
Go
package assets_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"path"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
"go.yaml.in/yaml/v4"
|
|
)
|
|
|
|
type assetSettings struct {
|
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
|
}
|
|
|
|
type noGeneration struct{ t *testing.T }
|
|
|
|
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
client.t.Error("resource preparation unexpectedly invoked generation")
|
|
return nil, fmt.Errorf("unexpected generation")
|
|
}
|
|
|
|
func loadNativeAssets(t *testing.T) (*configkit.Result[assetSettings], *promptkit.Engine) {
|
|
t.Helper()
|
|
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
|
definition := promptkit.Configure(configkit.Definition[assetSettings]{}, func(value *assetSettings) *promptkit.Settings { return &value.PromptKit })
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[assetSettings]{
|
|
Definition: definition,
|
|
Underlays: []configkit.Underlay{
|
|
promptkit.BundledOpenRouterUnderlay(),
|
|
{Inputs: []configkit.Input{{
|
|
Source: filesystem.File("notarius.yml"), Imports: true,
|
|
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."},
|
|
}}},
|
|
},
|
|
})
|
|
if err != nil {
|
|
var loadError *configkit.LoadError
|
|
if errors.As(err, &loadError) {
|
|
for _, diagnostic := range loadError.Diagnostics() {
|
|
t.Logf("%+v", diagnostic)
|
|
}
|
|
}
|
|
t.Fatal(err)
|
|
}
|
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(noGeneration{t}))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return result, engine
|
|
}
|
|
|
|
func TestEmbeddedPromptsPrepareWithNativeContracts(t *testing.T) {
|
|
t.Parallel()
|
|
_, engine := loadNativeAssets(t)
|
|
// Discover the actual declarations as well as the composed inventory, so an
|
|
// accidentally omitted import cannot silently remove a module's coverage.
|
|
err := fs.WalkDir(assets.FS(), ".", func(file string, entry fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if entry.IsDir() || path.Base(file) != "prompt.yaml" {
|
|
return nil
|
|
}
|
|
data, err := fs.ReadFile(assets.FS(), file)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var declared struct {
|
|
Prompts []promptkit.PromptSettings `yaml:"prompts"`
|
|
}
|
|
if err := yaml.Unmarshal(data, &declared); err != nil {
|
|
return err
|
|
}
|
|
if len(declared.Prompts) == 0 {
|
|
return fmt.Errorf("%s has no native prompts", file)
|
|
}
|
|
for _, prompt := range declared.Prompts {
|
|
t.Run(prompt.ID, func(t *testing.T) {
|
|
inputs := make(map[string]promptkit.ArtifactRef)
|
|
for _, input := range prompt.Inputs {
|
|
inputs[input.Name] = promptkit.Inline(input.Name + "-fixture")
|
|
}
|
|
request := promptkit.RunRequest{PromptID: "notarius/" + prompt.ID, Inputs: inputs, APIKey: "fixture-key"}
|
|
if prompt.DefaultProfile == nil {
|
|
request.ProfileID = "notarius/dnd-extraction"
|
|
}
|
|
prepared, err := engine.PrepareExecution(context.Background(), request)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer prepared.Discard()
|
|
details := prepared.Details()
|
|
if details.OutputContract.Format != promptkit.FormatJSON || details.OutputContract.ValidationMode != promptkit.ValidationJSONSchema || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil {
|
|
t.Fatalf("missing native schema contract: %#v", details.OutputContract)
|
|
}
|
|
// Preparation must load the real self-contained schema, not merely retain
|
|
// a filename or use a separate application schema registry.
|
|
schemaBytes, err := fs.ReadFile(assets.FS(), details.OutputContract.SchemaPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var expected any
|
|
if err := json.Unmarshal(schemaBytes, &expected); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
actualBytes, err := json.Marshal(details.StructuredOutput.JSONSchema.Schema)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var actual any
|
|
if err := json.Unmarshal(actualBytes, &actual); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(actual, expected) {
|
|
t.Fatal("prepared schema differs from the owned asset")
|
|
}
|
|
first := prompt.Messages[0]
|
|
if first.ContentFile == nil {
|
|
t.Fatal("expected a source-relative initial message")
|
|
}
|
|
content, err := fs.ReadFile(assets.FS(), path.Join(path.Dir(file), *first.ContentFile))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if first.Role == nil || len(details.Messages) == 0 || details.Messages[0].Role != *first.Role {
|
|
t.Fatal("declared message role was not preserved")
|
|
}
|
|
if !strings.Contains(string(content), "{{") && strings.TrimSpace(details.Messages[0].Content) != strings.TrimSpace(string(content)) {
|
|
t.Fatal("source-relative message content was not preserved")
|
|
}
|
|
})
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestEmbeddedProfileRetainsSelectedExecutionDefaults(t *testing.T) {
|
|
t.Parallel()
|
|
_, engine := loadNativeAssets(t)
|
|
profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if profile.Backend == nil || *profile.Backend != "openrouter" ||
|
|
profile.Model == nil || *profile.Model != "openai/gpt-5.6-luna" ||
|
|
profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 240 ||
|
|
profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil {
|
|
t.Fatalf("changed extraction defaults: %#v", profile)
|
|
}
|
|
}
|