Files
notarius/assets/package_test.go

215 lines
7.6 KiB
Go
Raw Normal View History

package assets_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"path"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"go.yaml.in/yaml/v4"
)
type assetSettings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
type noGeneration struct{ t *testing.T }
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.t.Error("resource preparation unexpectedly invoked generation")
return nil, fmt.Errorf("unexpected generation")
}
func loadNativeAssets(t *testing.T) (*configkit.Result[assetSettings], *promptkit.Engine) {
t.Helper()
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
definition := promptkit.Configure(configkit.Definition[assetSettings]{}, func(value *assetSettings) *promptkit.Settings { return &value.PromptKit })
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.NoSource()}, configkit.LayeredOptions[assetSettings]{
Definition: definition,
Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{
Source: filesystem.File("notarius.yml"), Imports: true,
Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."},
}}},
},
})
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
for _, diagnostic := range loadError.Diagnostics() {
t.Logf("%+v", diagnostic)
}
}
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(noGeneration{t}))
if err != nil {
t.Fatal(err)
}
return result, engine
}
func TestEmbeddedPromptsPrepareWithNativeContracts(t *testing.T) {
t.Parallel()
_, engine := loadNativeAssets(t)
// Discover the actual declarations as well as the composed inventory, so an
// accidentally omitted import cannot silently remove a module's coverage.
err := fs.WalkDir(assets.FS(), ".", func(file string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() || path.Base(file) != "prompt.yaml" {
return nil
}
data, err := fs.ReadFile(assets.FS(), file)
if err != nil {
return err
}
var declared struct {
Prompts []promptkit.PromptSettings `yaml:"prompts"`
}
if err := yaml.Unmarshal(data, &declared); err != nil {
return err
}
if len(declared.Prompts) == 0 {
return fmt.Errorf("%s has no native prompts", file)
}
for _, prompt := range declared.Prompts {
t.Run(prompt.ID, func(t *testing.T) {
inputs := make(map[string]promptkit.ArtifactRef)
for _, input := range prompt.Inputs {
inputs[input.Name] = promptkit.Inline(input.Name + "-fixture")
}
request := promptkit.RunRequest{PromptID: "notarius/" + prompt.ID, Inputs: inputs, APIKey: "fixture-key"}
if prompt.DefaultProfile == nil {
request.ProfileID = "notarius/dnd-extraction"
}
prepared, err := engine.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatal(err)
}
defer prepared.Discard()
details := prepared.Details()
if details.OutputContract.Format != promptkit.FormatJSON || details.OutputContract.ValidationMode != promptkit.ValidationJSONSchema || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil {
t.Fatalf("missing native schema contract: %#v", details.OutputContract)
}
// Preparation must load the real self-contained schema, not merely retain
// a filename or use a separate application schema registry.
schemaBytes, err := fs.ReadFile(assets.FS(), details.OutputContract.SchemaPath)
if err != nil {
t.Fatal(err)
}
var expected any
if err := json.Unmarshal(schemaBytes, &expected); err != nil {
t.Fatal(err)
}
actualBytes, err := json.Marshal(details.StructuredOutput.JSONSchema.Schema)
if err != nil {
t.Fatal(err)
}
var actual any
if err := json.Unmarshal(actualBytes, &actual); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actual, expected) {
t.Fatal("prepared schema differs from the owned asset")
}
first := prompt.Messages[0]
if first.ContentFile == nil {
t.Fatal("expected a source-relative initial message")
}
content, err := fs.ReadFile(assets.FS(), path.Join(path.Dir(file), *first.ContentFile))
if err != nil {
t.Fatal(err)
}
if first.Role == nil || len(details.Messages) == 0 || details.Messages[0].Role != *first.Role {
t.Fatal("declared message role was not preserved")
}
if !strings.Contains(string(content), "{{") && strings.TrimSpace(details.Messages[0].Content) != strings.TrimSpace(string(content)) {
t.Fatal("source-relative message content was not preserved")
}
})
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
func TestRegistryNormalizationPromptPairsRetainTheirNativeContracts(t *testing.T) {
_, engine := loadNativeAssets(t)
for _, test := range []struct {
id string
guidance string
rules []string
}{
{"notarius/dnd.npc_registry.normalize", "Preserve distinct individuals.", []string{"same individual", "unadorned proper name"}},
{"notarius/dnd.location_registry.normalize", "Keep parent and child places separate.", []string{"same physical place", "parent and child places"}},
} {
t.Run(test.id, func(t *testing.T) {
initial, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: test.id, APIKey: "test-only", Inputs: map[string]promptkit.ArtifactRef{
"candidates": promptkit.Inline(`{"candidate_number_range":{"first":1,"last":2},"candidates":[]}`),
"transcript": promptkit.Inline(`{"windows":[]}`),
},
})
if err != nil {
t.Fatal(err)
}
defer initial.Discard()
correction, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: test.id + ".repair", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(test.guidance)},
})
if err != nil {
t.Fatal(err)
}
defer correction.Discard()
first, next := initial.Details(), correction.Details()
if first.SelectedProfileID != "notarius/dnd-extraction" || next.SelectedProfileID != first.SelectedProfileID ||
first.StructuredOutput == nil || next.StructuredOutput == nil || !reflect.DeepEqual(first.OutputContract, next.OutputContract) {
t.Fatal("normalization pair lost its shared native defaults/output contract")
}
var system string
for _, message := range first.Messages {
if message.Role == "system" {
system += message.Content
}
}
for _, rule := range append(test.rules, "candidate_number_range", "start_unit_id", "canonical_candidate_number") {
if !strings.Contains(system, rule) {
t.Fatalf("missing enduring system principle %q", rule)
}
}
if len(next.Messages) != 1 || next.Messages[0].Role != "user" || !strings.Contains(next.Messages[0].Content, test.guidance) {
t.Fatal("correction did not supply only the findings and replacement assignment")
}
})
}
}
func TestEmbeddedProfileRetainsSelectedExecutionDefaults(t *testing.T) {
t.Parallel()
_, engine := loadNativeAssets(t)
profile, err := engine.InspectProfile(context.Background(), "notarius/dnd-extraction")
if err != nil {
t.Fatal(err)
}
if profile.Backend == nil || *profile.Backend != "openrouter" ||
profile.Model == nil || *profile.Model != "openai/gpt-5.6-luna" ||
profile.TimeoutSeconds == nil || *profile.TimeoutSeconds != 240 ||
profile.ServiceTier == nil || *profile.ServiceTier != "flex" || profile.ReasoningEffort != nil {
t.Fatalf("changed extraction defaults: %#v", profile)
}
}