2026-11-22 04:33:28 +00:00
|
|
|
package integration_test
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"fmt"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/assets"
|
|
|
|
|
"go.maximumdirect.net/configkit"
|
|
|
|
|
"go.maximumdirect.net/promptkit"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func nativeTestEngine(t *testing.T, client promptkit.LLMClient, profileIDs ...string) *promptkit.Engine {
|
|
|
|
|
t.Helper()
|
|
|
|
|
type settings struct {
|
|
|
|
|
PromptKit promptkit.Settings `yaml:"promptkit"`
|
|
|
|
|
}
|
|
|
|
|
profiles := make([]map[string]any, len(profileIDs))
|
|
|
|
|
for index, id := range profileIDs {
|
|
|
|
|
profiles[index] = map[string]any{"id": id, "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false}
|
|
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
content, err := json.Marshal(map[string]any{"promptkit": map[string]any{
|
|
|
|
|
"profiles": profiles,
|
|
|
|
|
"collections": map[string]any{"notarius": map[string]any{"profiles": []map[string]any{
|
|
|
|
|
{"id": "dnd-extraction", "endpoint": "https://example.invalid/v1", "model": "integration-test-model", "api_key_required": false},
|
|
|
|
|
}}},
|
|
|
|
|
}})
|
2026-11-22 04:33:28 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
|
|
|
|
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
|
|
|
|
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("native provider fixture", content)}, configkit.LayeredOptions[settings]{
|
|
|
|
|
Definition: definition,
|
|
|
|
|
Underlays: []configkit.Underlay{
|
|
|
|
|
promptkit.BundledOpenRouterUnderlay(),
|
|
|
|
|
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
|
|
|
|
},
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return engine
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type nativeSequenceClient struct {
|
|
|
|
|
mu sync.Mutex
|
|
|
|
|
responses []string
|
|
|
|
|
requests []promptkit.GenerateRequest
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (client *nativeSequenceClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
client.mu.Lock()
|
|
|
|
|
defer client.mu.Unlock()
|
|
|
|
|
if err := ctx.Err(); err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
index := len(client.requests)
|
|
|
|
|
client.requests = append(client.requests, request)
|
|
|
|
|
if index >= len(client.responses) {
|
|
|
|
|
return nil, fmt.Errorf("unexpected provider request %d", index+1)
|
|
|
|
|
}
|
|
|
|
|
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func nativePromptText(request promptkit.GenerateRequest) string {
|
|
|
|
|
var content []string
|
|
|
|
|
for _, message := range request.Prompt.Messages {
|
|
|
|
|
content = append(content, message.Content)
|
|
|
|
|
}
|
|
|
|
|
return strings.Join(content, "\n")
|
|
|
|
|
}
|
2026-11-22 04:39:55 +00:00
|
|
|
|
|
|
|
|
func nativeSchemaHasProperty(request promptkit.GenerateRequest, property string) bool {
|
|
|
|
|
if request.StructuredOutput == nil || request.StructuredOutput.JSONSchema == nil {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
content, err := json.Marshal(request.StructuredOutput.JSONSchema.Schema)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
var schema struct {
|
|
|
|
|
Properties map[string]json.RawMessage `json:"properties"`
|
|
|
|
|
}
|
|
|
|
|
if json.Unmarshal(content, &schema) != nil {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
_, found := schema.Properties[property]
|
|
|
|
|
return found
|
|
|
|
|
}
|