Add captured framework configuration inspection
This commit is contained in:
139
internal/adapter/cli/framework_config.go
Normal file
139
internal/adapter/cli/framework_config.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
type frameworkConfigurationConfig struct {
|
||||
mode, configPath, configOverlay, schemaDir string
|
||||
configExplicit bool
|
||||
overrides []configkit.Override
|
||||
diagnostics []configkit.Diagnostic
|
||||
}
|
||||
|
||||
type frameworkConfigurationSource struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Layer configkit.Layer `json:"layer"`
|
||||
Kind configkit.SourceKind `json:"kind"`
|
||||
UnderlayIndex int `json:"underlay_index"`
|
||||
Path string `json:"path,omitempty"`
|
||||
ImportedFrom *frameworkConfigurationLocation `json:"imported_from,omitempty"`
|
||||
}
|
||||
|
||||
type frameworkConfigurationLocation struct {
|
||||
SourceID string `json:"source_id"`
|
||||
Line int `json:"line"`
|
||||
Column int `json:"column"`
|
||||
}
|
||||
|
||||
func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Writer) int {
|
||||
cfg, err := parseFrameworkConfigurationArgs(args)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect configuration parse", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
settings, err := resolveAppSettingsForFrameworkConfiguration(cfg)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect configuration", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
engine, err := newEngine(settings)
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "engine", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
view, err := engine.InspectConfiguration()
|
||||
if err != nil {
|
||||
writeApplicationError(stderr, "inspect configuration", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if err := writeApplicationWarnings(stderr, view.Diagnostics()); err != nil {
|
||||
return ExitRuntimeError
|
||||
}
|
||||
if cfg.mode == "validate" {
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
switch cfg.mode {
|
||||
case "show":
|
||||
err = view.WriteEffective(stdout)
|
||||
case "sources":
|
||||
sources := frameworkConfigurationSources(view.Sources())
|
||||
err = json.NewEncoder(stdout).Encode(sources)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "inspect configuration output error: %v\n", err)
|
||||
return ExitRuntimeError
|
||||
}
|
||||
return ExitOK
|
||||
}
|
||||
|
||||
func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConfig, error) {
|
||||
if len(args) == 0 {
|
||||
return nil, errors.New("configuration operation is required")
|
||||
}
|
||||
cfg := &frameworkConfigurationConfig{mode: args[0]}
|
||||
switch cfg.mode {
|
||||
case "validate", "show", "sources":
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode)
|
||||
}
|
||||
|
||||
fs := flag.NewFlagSet("inspect configuration "+cfg.mode, flag.ContinueOnError)
|
||||
fs.SetOutput(io.Discard)
|
||||
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||
var promptkitSources promptKitSourceFlagValue
|
||||
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||
}
|
||||
cfg.configExplicit = flagWasSet(fs, "config")
|
||||
cfg.overrides = applicationOverrides(fs)
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func resolveAppSettingsForFrameworkConfiguration(cfg *frameworkConfigurationConfig) (engineSettings, error) {
|
||||
settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides)
|
||||
if err != nil {
|
||||
return engineSettings{}, err
|
||||
}
|
||||
cfg.diagnostics = diagnostics
|
||||
return engineSettings{
|
||||
promptkitSources: settings.PromptKitSources,
|
||||
schemaDir: settings.SchemaDir,
|
||||
transientRetries: settings.TransientRetries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func frameworkConfigurationSources(sources []configkit.SourceInfo) []frameworkConfigurationSource {
|
||||
result := make([]frameworkConfigurationSource, 0, len(sources))
|
||||
for _, source := range sources {
|
||||
item := frameworkConfigurationSource{
|
||||
ID: source.ID, Label: source.Label, Layer: source.Layer,
|
||||
Kind: source.Kind, UnderlayIndex: source.UnderlayIndex, Path: source.Path,
|
||||
}
|
||||
if source.ImportedFrom != nil {
|
||||
item.ImportedFrom = &frameworkConfigurationLocation{
|
||||
SourceID: source.ImportedFrom.SourceID,
|
||||
Line: source.ImportedFrom.Line,
|
||||
Column: source.ImportedFrom.Column,
|
||||
}
|
||||
}
|
||||
result = append(result, item)
|
||||
}
|
||||
return result
|
||||
}
|
||||
193
internal/adapter/cli/framework_config_test.go
Normal file
193
internal/adapter/cli/framework_config_test.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, "{}\n")
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"configuration", "validate", "--config", configPath,
|
||||
})
|
||||
if code != ExitOK || stdout != "" || stderr != "" {
|
||||
t.Fatalf("source-free validation failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
|
||||
t.Setenv("SCRIPTORIUM_FRAMEWORK_KEY", "")
|
||||
root := t.TempDir()
|
||||
frameworkFile := filepath.Join(root, "framework.yml")
|
||||
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||
local:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
|
||||
profiles:
|
||||
protected:
|
||||
backend: local
|
||||
model: offline-model
|
||||
api_key_required: true
|
||||
`)
|
||||
configPath = writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||
for _, mode := range []string{"validate", "show", "sources"} {
|
||||
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{
|
||||
"configuration", mode, "--config", configPath,
|
||||
})
|
||||
if code != ExitOK || stderr != "" {
|
||||
t.Fatalf("%s required a prompt or credential: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr)
|
||||
}
|
||||
if mode == "validate" && stdout != "" {
|
||||
t.Fatalf("validation wrote success output: %q", stdout)
|
||||
}
|
||||
if mode == "show" && (!strings.Contains(stdout, "profiles:") || !strings.Contains(stdout, "offline-model")) {
|
||||
t.Fatalf("effective framework YAML is missing: %q", stdout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
importDir := filepath.Join(root, "catalog")
|
||||
if err := os.MkdirAll(importDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit:
|
||||
imports:
|
||||
- path: catalog/fragment.yml
|
||||
backends:
|
||||
local:
|
||||
endpoint: http://127.0.0.1:1/v1
|
||||
profiles:
|
||||
selected:
|
||||
backend: local
|
||||
model: selected-model
|
||||
`)
|
||||
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
|
||||
imported:
|
||||
model: imported-model
|
||||
`)
|
||||
directory := t.TempDir()
|
||||
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
|
||||
directory:
|
||||
model: directory-model
|
||||
`)
|
||||
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
|
||||
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", configuredMissing))
|
||||
args := []string{
|
||||
"configuration", "sources", "--config", configPath,
|
||||
"--promptkit-source", mainPath,
|
||||
"--promptkit-source", directory,
|
||||
}
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") {
|
||||
t.Fatalf("framework sources command failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
var sources []frameworkConfigurationSource
|
||||
if err := json.Unmarshal([]byte(stdout), &sources); err != nil {
|
||||
t.Fatalf("decode framework source DTOs: %v\n%s", err, stdout)
|
||||
}
|
||||
var rawSources []map[string]any
|
||||
if err := json.Unmarshal([]byte(stdout), &rawSources); err != nil {
|
||||
t.Fatalf("decode framework source fields: %v", err)
|
||||
}
|
||||
mainIndex := findFrameworkSource(t, sources, mainPath)
|
||||
fragmentIndex := findFrameworkSource(t, sources, fragmentPath)
|
||||
directoryIndex := findFrameworkSource(t, sources, directoryPath)
|
||||
main := sources[mainIndex]
|
||||
fragment := sources[fragmentIndex]
|
||||
directorySource := sources[directoryIndex]
|
||||
if !(mainIndex < fragmentIndex && fragmentIndex < directoryIndex) {
|
||||
t.Fatalf("captured source order changed: main=%d fragment=%d directory=%d", mainIndex, fragmentIndex, directoryIndex)
|
||||
}
|
||||
if main.ID == "" || main.Label == "" || main.Layer != configkit.LayerUnderlay || main.Kind != configkit.SourceKindFile || main.UnderlayIndex < 1 {
|
||||
t.Fatalf("explicit file source fields are incomplete: %+v", main)
|
||||
}
|
||||
for _, key := range []string{"id", "label", "layer", "kind", "underlay_index", "path"} {
|
||||
if _, exists := rawSources[mainIndex][key]; !exists {
|
||||
t.Fatalf("source DTO omitted %q: %#v", key, rawSources[mainIndex])
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"ID", "Label", "Layer", "Kind", "UnderlayIndex", "filesystem"} {
|
||||
if _, exists := rawSources[mainIndex][key]; exists {
|
||||
t.Fatalf("source DTO exposed implementation field %q: %#v", key, rawSources[mainIndex])
|
||||
}
|
||||
}
|
||||
if fragment.Layer != configkit.LayerUnderlay || fragment.Kind != configkit.SourceKindFile || fragment.ImportedFrom == nil ||
|
||||
fragment.ImportedFrom.SourceID != main.ID || fragment.ImportedFrom.Line < 1 || fragment.ImportedFrom.Column < 1 {
|
||||
t.Fatalf("import source fields are incomplete: %+v", fragment)
|
||||
}
|
||||
if importedFrom, ok := rawSources[fragmentIndex]["imported_from"].(map[string]any); !ok || importedFrom["source_id"] != main.ID || importedFrom["line"] == nil || importedFrom["column"] == nil {
|
||||
t.Fatalf("import location DTO fields are incomplete: %#v", rawSources[fragmentIndex])
|
||||
}
|
||||
if directorySource.Layer != configkit.LayerUnderlay || directorySource.Kind != configkit.SourceKindFile || directorySource.UnderlayIndex <= main.UnderlayIndex {
|
||||
t.Fatalf("directory-selected source fields/order are incomplete: %+v (main %+v)", directorySource, main)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
|
||||
unused:
|
||||
"1":
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||
"configuration", "show", "--promptkit-source", frameworkFile,
|
||||
})
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors(t *testing.T) {
|
||||
configPath := writeAppConfigFile(t, "{}\n")
|
||||
for _, args := range [][]string{
|
||||
{"configuration"},
|
||||
{"configuration", "explain"},
|
||||
{"configuration", "show", "unexpected"},
|
||||
{"configuration", "sources", "--format", "json"},
|
||||
} {
|
||||
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||
t.Fatalf("invalid inspection arguments were accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
writerError := errors.New("synthetic framework output failure")
|
||||
var stderr bytes.Buffer
|
||||
code := inspectCommand([]string{"configuration", "show", "--config", configPath}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
|
||||
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
||||
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {
|
||||
t.Helper()
|
||||
for index, source := range sources {
|
||||
if filepath.Clean(source.Path) == filepath.Clean(path) {
|
||||
return index
|
||||
}
|
||||
}
|
||||
t.Fatalf("selected source path %q was not present: %+v", path, sources)
|
||||
return -1
|
||||
}
|
||||
|
||||
type frameworkConfigurationFailingWriter struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (writer frameworkConfigurationFailingWriter) Write([]byte) (int, error) {
|
||||
return 0, writer.err
|
||||
}
|
||||
@@ -39,6 +39,9 @@ func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
||||
if args[0] == "profile" {
|
||||
return inspectProfileCommand(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] == "configuration" {
|
||||
return inspectFrameworkConfigurationCommand(args[1:], stdout, stderr)
|
||||
}
|
||||
if args[0] != "prompt" {
|
||||
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
|
||||
return ExitRuntimeError
|
||||
|
||||
Reference in New Issue
Block a user