194 lines
7.4 KiB
Go
194 lines
7.4 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
)
|
|
|
|
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, "{}\n")
|
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
|
"configuration", "validate", "--config", configPath,
|
|
})
|
|
if code != ExitOK || stdout != "" || stderr != "" {
|
|
t.Fatalf("source-free validation failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
|
|
t.Setenv("SCRIPTORIUM_FRAMEWORK_KEY", "")
|
|
root := t.TempDir()
|
|
frameworkFile := filepath.Join(root, "framework.yml")
|
|
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
|
local:
|
|
endpoint: http://127.0.0.1:1/v1
|
|
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
|
|
profiles:
|
|
protected:
|
|
backend: local
|
|
model: offline-model
|
|
api_key_required: true
|
|
`)
|
|
configPath = writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
|
for _, mode := range []string{"validate", "show", "sources"} {
|
|
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{
|
|
"configuration", mode, "--config", configPath,
|
|
})
|
|
if code != ExitOK || stderr != "" {
|
|
t.Fatalf("%s required a prompt or credential: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr)
|
|
}
|
|
if mode == "validate" && stdout != "" {
|
|
t.Fatalf("validation wrote success output: %q", stdout)
|
|
}
|
|
if mode == "show" && (!strings.Contains(stdout, "profiles:") || !strings.Contains(stdout, "offline-model")) {
|
|
t.Fatalf("effective framework YAML is missing: %q", stdout)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T) {
|
|
root := t.TempDir()
|
|
importDir := filepath.Join(root, "catalog")
|
|
if err := os.MkdirAll(importDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit:
|
|
imports:
|
|
- path: catalog/fragment.yml
|
|
backends:
|
|
local:
|
|
endpoint: http://127.0.0.1:1/v1
|
|
profiles:
|
|
selected:
|
|
backend: local
|
|
model: selected-model
|
|
`)
|
|
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
|
|
imported:
|
|
model: imported-model
|
|
`)
|
|
directory := t.TempDir()
|
|
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
|
|
directory:
|
|
model: directory-model
|
|
`)
|
|
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
|
|
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", configuredMissing))
|
|
args := []string{
|
|
"configuration", "sources", "--config", configPath,
|
|
"--promptkit-source", mainPath,
|
|
"--promptkit-source", directory,
|
|
}
|
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
|
if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") {
|
|
t.Fatalf("framework sources command failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
var sources []frameworkConfigurationSource
|
|
if err := json.Unmarshal([]byte(stdout), &sources); err != nil {
|
|
t.Fatalf("decode framework source DTOs: %v\n%s", err, stdout)
|
|
}
|
|
var rawSources []map[string]any
|
|
if err := json.Unmarshal([]byte(stdout), &rawSources); err != nil {
|
|
t.Fatalf("decode framework source fields: %v", err)
|
|
}
|
|
mainIndex := findFrameworkSource(t, sources, mainPath)
|
|
fragmentIndex := findFrameworkSource(t, sources, fragmentPath)
|
|
directoryIndex := findFrameworkSource(t, sources, directoryPath)
|
|
main := sources[mainIndex]
|
|
fragment := sources[fragmentIndex]
|
|
directorySource := sources[directoryIndex]
|
|
if !(mainIndex < fragmentIndex && fragmentIndex < directoryIndex) {
|
|
t.Fatalf("captured source order changed: main=%d fragment=%d directory=%d", mainIndex, fragmentIndex, directoryIndex)
|
|
}
|
|
if main.ID == "" || main.Label == "" || main.Layer != configkit.LayerUnderlay || main.Kind != configkit.SourceKindFile || main.UnderlayIndex < 1 {
|
|
t.Fatalf("explicit file source fields are incomplete: %+v", main)
|
|
}
|
|
for _, key := range []string{"id", "label", "layer", "kind", "underlay_index", "path"} {
|
|
if _, exists := rawSources[mainIndex][key]; !exists {
|
|
t.Fatalf("source DTO omitted %q: %#v", key, rawSources[mainIndex])
|
|
}
|
|
}
|
|
for _, key := range []string{"ID", "Label", "Layer", "Kind", "UnderlayIndex", "filesystem"} {
|
|
if _, exists := rawSources[mainIndex][key]; exists {
|
|
t.Fatalf("source DTO exposed implementation field %q: %#v", key, rawSources[mainIndex])
|
|
}
|
|
}
|
|
if fragment.Layer != configkit.LayerUnderlay || fragment.Kind != configkit.SourceKindFile || fragment.ImportedFrom == nil ||
|
|
fragment.ImportedFrom.SourceID != main.ID || fragment.ImportedFrom.Line < 1 || fragment.ImportedFrom.Column < 1 {
|
|
t.Fatalf("import source fields are incomplete: %+v", fragment)
|
|
}
|
|
if importedFrom, ok := rawSources[fragmentIndex]["imported_from"].(map[string]any); !ok || importedFrom["source_id"] != main.ID || importedFrom["line"] == nil || importedFrom["column"] == nil {
|
|
t.Fatalf("import location DTO fields are incomplete: %#v", rawSources[fragmentIndex])
|
|
}
|
|
if directorySource.Layer != configkit.LayerUnderlay || directorySource.Kind != configkit.SourceKindFile || directorySource.UnderlayIndex <= main.UnderlayIndex {
|
|
t.Fatalf("directory-selected source fields/order are incomplete: %+v (main %+v)", directorySource, main)
|
|
}
|
|
}
|
|
|
|
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
|
|
root := t.TempDir()
|
|
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
|
|
unused:
|
|
"1":
|
|
messages:
|
|
- role: user
|
|
content: "{{"
|
|
output:
|
|
format: text
|
|
validation_mode: none
|
|
`)
|
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
|
"configuration", "show", "--promptkit-source", frameworkFile,
|
|
})
|
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
|
t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
}
|
|
|
|
func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors(t *testing.T) {
|
|
configPath := writeAppConfigFile(t, "{}\n")
|
|
for _, args := range [][]string{
|
|
{"configuration"},
|
|
{"configuration", "explain"},
|
|
{"configuration", "show", "unexpected"},
|
|
{"configuration", "sources", "--format", "json"},
|
|
} {
|
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
|
t.Fatalf("invalid inspection arguments were accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
|
}
|
|
}
|
|
|
|
writerError := errors.New("synthetic framework output failure")
|
|
var stderr bytes.Buffer
|
|
code := inspectCommand([]string{"configuration", "show", "--config", configPath}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
|
|
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
|
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {
|
|
t.Helper()
|
|
for index, source := range sources {
|
|
if filepath.Clean(source.Path) == filepath.Clean(path) {
|
|
return index
|
|
}
|
|
}
|
|
t.Fatalf("selected source path %q was not present: %+v", path, sources)
|
|
return -1
|
|
}
|
|
|
|
type frameworkConfigurationFailingWriter struct {
|
|
err error
|
|
}
|
|
|
|
func (writer frameworkConfigurationFailingWriter) Write([]byte) (int, error) {
|
|
return 0, writer.err
|
|
}
|