Files
scriptorium/internal/adapter/cli/framework_config_test.go

194 lines
7.4 KiB
Go

package cli
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
)
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"configuration", "validate", "--config", configPath,
})
if code != ExitOK || stdout != "" || stderr != "" {
t.Fatalf("source-free validation failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
t.Setenv("SCRIPTORIUM_FRAMEWORK_KEY", "")
root := t.TempDir()
frameworkFile := filepath.Join(root, "framework.yml")
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
local:
endpoint: http://127.0.0.1:1/v1
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
profiles:
protected:
backend: local
model: offline-model
api_key_required: true
`)
configPath = writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
for _, mode := range []string{"validate", "show", "sources"} {
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{
"configuration", mode, "--config", configPath,
})
if code != ExitOK || stderr != "" {
t.Fatalf("%s required a prompt or credential: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr)
}
if mode == "validate" && stdout != "" {
t.Fatalf("validation wrote success output: %q", stdout)
}
if mode == "show" && (!strings.Contains(stdout, "profiles:") || !strings.Contains(stdout, "offline-model")) {
t.Fatalf("effective framework YAML is missing: %q", stdout)
}
}
}
func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T) {
root := t.TempDir()
importDir := filepath.Join(root, "catalog")
if err := os.MkdirAll(importDir, 0o755); err != nil {
t.Fatal(err)
}
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit:
imports:
- path: catalog/fragment.yml
backends:
local:
endpoint: http://127.0.0.1:1/v1
profiles:
selected:
backend: local
model: selected-model
`)
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
imported:
model: imported-model
`)
directory := t.TempDir()
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
directory:
model: directory-model
`)
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", configuredMissing))
args := []string{
"configuration", "sources", "--config", configPath,
"--promptkit-source", mainPath,
"--promptkit-source", directory,
}
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") {
t.Fatalf("framework sources command failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
var sources []frameworkConfigurationSource
if err := json.Unmarshal([]byte(stdout), &sources); err != nil {
t.Fatalf("decode framework source DTOs: %v\n%s", err, stdout)
}
var rawSources []map[string]any
if err := json.Unmarshal([]byte(stdout), &rawSources); err != nil {
t.Fatalf("decode framework source fields: %v", err)
}
mainIndex := findFrameworkSource(t, sources, mainPath)
fragmentIndex := findFrameworkSource(t, sources, fragmentPath)
directoryIndex := findFrameworkSource(t, sources, directoryPath)
main := sources[mainIndex]
fragment := sources[fragmentIndex]
directorySource := sources[directoryIndex]
if !(mainIndex < fragmentIndex && fragmentIndex < directoryIndex) {
t.Fatalf("captured source order changed: main=%d fragment=%d directory=%d", mainIndex, fragmentIndex, directoryIndex)
}
if main.ID == "" || main.Label == "" || main.Layer != configkit.LayerUnderlay || main.Kind != configkit.SourceKindFile || main.UnderlayIndex < 1 {
t.Fatalf("explicit file source fields are incomplete: %+v", main)
}
for _, key := range []string{"id", "label", "layer", "kind", "underlay_index", "path"} {
if _, exists := rawSources[mainIndex][key]; !exists {
t.Fatalf("source DTO omitted %q: %#v", key, rawSources[mainIndex])
}
}
for _, key := range []string{"ID", "Label", "Layer", "Kind", "UnderlayIndex", "filesystem"} {
if _, exists := rawSources[mainIndex][key]; exists {
t.Fatalf("source DTO exposed implementation field %q: %#v", key, rawSources[mainIndex])
}
}
if fragment.Layer != configkit.LayerUnderlay || fragment.Kind != configkit.SourceKindFile || fragment.ImportedFrom == nil ||
fragment.ImportedFrom.SourceID != main.ID || fragment.ImportedFrom.Line < 1 || fragment.ImportedFrom.Column < 1 {
t.Fatalf("import source fields are incomplete: %+v", fragment)
}
if importedFrom, ok := rawSources[fragmentIndex]["imported_from"].(map[string]any); !ok || importedFrom["source_id"] != main.ID || importedFrom["line"] == nil || importedFrom["column"] == nil {
t.Fatalf("import location DTO fields are incomplete: %#v", rawSources[fragmentIndex])
}
if directorySource.Layer != configkit.LayerUnderlay || directorySource.Kind != configkit.SourceKindFile || directorySource.UnderlayIndex <= main.UnderlayIndex {
t.Fatalf("directory-selected source fields/order are incomplete: %+v (main %+v)", directorySource, main)
}
}
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
root := t.TempDir()
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
unused:
"1":
messages:
- role: user
content: "{{"
output:
format: text
validation_mode: none
`)
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
"configuration", "show", "--promptkit-source", frameworkFile,
})
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
}
func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors(t *testing.T) {
configPath := writeAppConfigFile(t, "{}\n")
for _, args := range [][]string{
{"configuration"},
{"configuration", "explain"},
{"configuration", "show", "unexpected"},
{"configuration", "sources", "--format", "json"},
} {
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
if code != ExitRuntimeError || stdout != "" || stderr == "" {
t.Fatalf("invalid inspection arguments were accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
}
}
writerError := errors.New("synthetic framework output failure")
var stderr bytes.Buffer
code := inspectCommand([]string{"configuration", "show", "--config", configPath}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
}
}
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {
t.Helper()
for index, source := range sources {
if filepath.Clean(source.Path) == filepath.Clean(path) {
return index
}
}
t.Fatalf("selected source path %q was not present: %+v", path, sources)
return -1
}
type frameworkConfigurationFailingWriter struct {
err error
}
func (writer frameworkConfigurationFailingWriter) Write([]byte) (int, error) {
return 0, writer.err
}