Files
notarius/internal/modules/integration/native_provider_test.go

114 lines
3.7 KiB
Go
Raw Normal View History

package integration_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func nativeTestEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine {
t.Helper()
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
profiles := make([]map[string]any, len(profileIDs))
for index, id := range profileIDs {
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false}
}
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{
"profiles": profiles,
"collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{
{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false},
}}},
}})
if err != nil {
t.Fatal(err)
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("native provider fixture", content)}, configkit.LayeredOptions[settings]{
Definition: definition,
Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
},
})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
return engine
}
type nativeSequenceClient struct {
mu sync.Mutex
responses []string
requests []promptkit.GenerateRequest
}
func (client *nativeSequenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.mu.Lock()
defer client.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, err
}
index := len(client.requests)
client.requests = append(client.requests, request)
if index >= len(client.responses) {
return nil, fmt.Errorf("unexpected provider request %d", index+1)
}
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
}
func nativePromptText(request promptkit.GenerateRequest) string {
var content []string
for _, message := range request.Prompt.Messages {
content = append(content, message.Content)
}
return strings.Join(content, "\n")
}
func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string) bool {
if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil {
return false
}
content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema)
if err != nil {
return false
}
var schema struct {
Properties map[string]json.RawMessage `json:"properties"`
}
if json.Unmarshal(content, &schema) != nil {
return false
}
_, found := schema.Properties[property]
return found
}
func nativeJSONInput(request promptkit.GenerateRequest, field string) ([]byte, error) {
for _, message := range request.Prompt.Messages {
for offset := range len(message.Content) {
if message.Content[offset] != '{' {
continue
}
decoder := json.NewDecoder(strings.NewReader(message.Content[offset:]))
var object map[string]json.RawMessage
if decoder.Decode(&object) == nil && object[field] != nil {
return []byte(message.Content[offset : offset+int(decoder.InputOffset())]), nil
}
}
}
return nil, fmt.Errorf("native request has no JSON input with field %q", field)
}