Add captured framework configuration inspection
This commit is contained in:
@@ -3,8 +3,8 @@
|
|||||||
## Status, Baseline, And Authority
|
## Status, Baseline, And Authority
|
||||||
|
|
||||||
Planned on 2026-10-04 against Scriptorium
|
Planned on 2026-10-04 against Scriptorium
|
||||||
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–8 are complete; stages
|
`82b0f231b6ba71328f65f9a501143b2eedbe5d9c`. Stages 1–9 are complete; stages
|
||||||
9–13 are pending.
|
10–13 are pending.
|
||||||
This plan is for a Luna coding agent: implement the stages in sequential order,
|
This plan is for a Luna coding agent: implement the stages in sequential order,
|
||||||
normally one stage per implementation prompt. The
|
normally one stage per implementation prompt. The
|
||||||
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
[feature roadmap](promptkit.md) owns the target state, scope, and confirmed
|
||||||
@@ -522,7 +522,16 @@ definition plumbing are protected at the actual application boundary.
|
|||||||
|
|
||||||
## Stage 9: Add Captured Framework Validate, Show, And Sources Commands
|
## Stage 9: Add Captured Framework Validate, Show, And Sources Commands
|
||||||
|
|
||||||
Status: Pending. Depends on Stage 8.
|
Status: Complete. `inspect configuration validate`, `show`, and `sources` now
|
||||||
|
load application settings, construct one engine, and use its captured public
|
||||||
|
inspection view without preparing a request or resolving credentials. Effective
|
||||||
|
output remains PromptKit's native YAML; source JSON uses the selected DTO fields
|
||||||
|
and preserves imported locations and source order. Tests cover source-free and
|
||||||
|
credential-required configurations, invalid final definitions, source layering
|
||||||
|
and imports, stdout/stderr separation, positional rejection, and writer
|
||||||
|
failures. Focused inspection tests, `go test ./...`, `go test -race ./...`,
|
||||||
|
`go vet ./...`, the CLI build, `gofmt`, and `git diff --check` pass. Literal-path
|
||||||
|
explanations and their redaction coverage remain for Stage 10.
|
||||||
|
|
||||||
Files: `internal/adapter/cli/inspect.go`, a small
|
Files: `internal/adapter/cli/inspect.go`, a small
|
||||||
`framework_config.go` and `framework_config_test.go` in that package.
|
`framework_config.go` and `framework_config_test.go` in that package.
|
||||||
|
|||||||
139
internal/adapter/cli/framework_config.go
Normal file
139
internal/adapter/cli/framework_config.go
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type frameworkConfigurationConfig struct {
|
||||||
|
mode, configPath, configOverlay, schemaDir string
|
||||||
|
configExplicit bool
|
||||||
|
overrides []configkit.Override
|
||||||
|
diagnostics []configkit.Diagnostic
|
||||||
|
}
|
||||||
|
|
||||||
|
type frameworkConfigurationSource struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Layer configkit.Layer `json:"layer"`
|
||||||
|
Kind configkit.SourceKind `json:"kind"`
|
||||||
|
UnderlayIndex int `json:"underlay_index"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
ImportedFrom *frameworkConfigurationLocation `json:"imported_from,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type frameworkConfigurationLocation struct {
|
||||||
|
SourceID string `json:"source_id"`
|
||||||
|
Line int `json:"line"`
|
||||||
|
Column int `json:"column"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspectFrameworkConfigurationCommand(args []string, stdout, stderr io.Writer) int {
|
||||||
|
cfg, err := parseFrameworkConfigurationArgs(args)
|
||||||
|
if err != nil {
|
||||||
|
writeApplicationError(stderr, "inspect configuration parse", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
settings, err := resolveAppSettingsForFrameworkConfiguration(cfg)
|
||||||
|
if err != nil {
|
||||||
|
writeApplicationError(stderr, "inspect configuration", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if err := writeApplicationWarnings(stderr, cfg.diagnostics); err != nil {
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
engine, err := newEngine(settings)
|
||||||
|
if err != nil {
|
||||||
|
writeApplicationError(stderr, "engine", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
view, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
writeApplicationError(stderr, "inspect configuration", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if err := writeApplicationWarnings(stderr, view.Diagnostics()); err != nil {
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
if cfg.mode == "validate" {
|
||||||
|
return ExitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
switch cfg.mode {
|
||||||
|
case "show":
|
||||||
|
err = view.WriteEffective(stdout)
|
||||||
|
case "sources":
|
||||||
|
sources := frameworkConfigurationSources(view.Sources())
|
||||||
|
err = json.NewEncoder(stdout).Encode(sources)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "inspect configuration output error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
return ExitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseFrameworkConfigurationArgs(args []string) (*frameworkConfigurationConfig, error) {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return nil, errors.New("configuration operation is required")
|
||||||
|
}
|
||||||
|
cfg := &frameworkConfigurationConfig{mode: args[0]}
|
||||||
|
switch cfg.mode {
|
||||||
|
case "validate", "show", "sources":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown configuration operation %q", cfg.mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
fs := flag.NewFlagSet("inspect configuration "+cfg.mode, flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
registerConfigSelectionFlags(fs, &cfg.configPath, &cfg.configOverlay)
|
||||||
|
var promptkitSources promptKitSourceFlagValue
|
||||||
|
fs.Var(&promptkitSources, "promptkit-source", "selected PromptKit source path (repeatable)")
|
||||||
|
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||||
|
if err := fs.Parse(args[1:]); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if fs.NArg() != 0 {
|
||||||
|
return nil, fmt.Errorf("unexpected positional args: %v", fs.Args())
|
||||||
|
}
|
||||||
|
cfg.configExplicit = flagWasSet(fs, "config")
|
||||||
|
cfg.overrides = applicationOverrides(fs)
|
||||||
|
return cfg, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveAppSettingsForFrameworkConfiguration(cfg *frameworkConfigurationConfig) (engineSettings, error) {
|
||||||
|
settings, diagnostics, err := loadApplicationSettings(cfg.configPath, cfg.configOverlay, cfg.configExplicit, cfg.overrides)
|
||||||
|
if err != nil {
|
||||||
|
return engineSettings{}, err
|
||||||
|
}
|
||||||
|
cfg.diagnostics = diagnostics
|
||||||
|
return engineSettings{
|
||||||
|
promptkitSources: settings.PromptKitSources,
|
||||||
|
schemaDir: settings.SchemaDir,
|
||||||
|
transientRetries: settings.TransientRetries,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func frameworkConfigurationSources(sources []configkit.SourceInfo) []frameworkConfigurationSource {
|
||||||
|
result := make([]frameworkConfigurationSource, 0, len(sources))
|
||||||
|
for _, source := range sources {
|
||||||
|
item := frameworkConfigurationSource{
|
||||||
|
ID: source.ID, Label: source.Label, Layer: source.Layer,
|
||||||
|
Kind: source.Kind, UnderlayIndex: source.UnderlayIndex, Path: source.Path,
|
||||||
|
}
|
||||||
|
if source.ImportedFrom != nil {
|
||||||
|
item.ImportedFrom = &frameworkConfigurationLocation{
|
||||||
|
SourceID: source.ImportedFrom.SourceID,
|
||||||
|
Line: source.ImportedFrom.Line,
|
||||||
|
Column: source.ImportedFrom.Column,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = append(result, item)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
193
internal/adapter/cli/framework_config_test.go
Normal file
193
internal/adapter/cli/framework_config_test.go
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFrameworkConfigurationCommandsNeedNoPromptOrCredential(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"configuration", "validate", "--config", configPath,
|
||||||
|
})
|
||||||
|
if code != ExitOK || stdout != "" || stderr != "" {
|
||||||
|
t.Fatalf("source-free validation failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("SCRIPTORIUM_FRAMEWORK_KEY", "")
|
||||||
|
root := t.TempDir()
|
||||||
|
frameworkFile := filepath.Join(root, "framework.yml")
|
||||||
|
writeFrameworkSourceFile(t, root, "framework.yml", `backends:
|
||||||
|
local:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
api_key_env: SCRIPTORIUM_FRAMEWORK_KEY
|
||||||
|
profiles:
|
||||||
|
protected:
|
||||||
|
backend: local
|
||||||
|
model: offline-model
|
||||||
|
api_key_required: true
|
||||||
|
`)
|
||||||
|
configPath = writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", frameworkFile))
|
||||||
|
for _, mode := range []string{"validate", "show", "sources"} {
|
||||||
|
code, stdout, stderr = runCLICommand(t, inspectCommand, []string{
|
||||||
|
"configuration", mode, "--config", configPath,
|
||||||
|
})
|
||||||
|
if code != ExitOK || stderr != "" {
|
||||||
|
t.Fatalf("%s required a prompt or credential: code=%d stdout=%q stderr=%q", mode, code, stdout, stderr)
|
||||||
|
}
|
||||||
|
if mode == "validate" && stdout != "" {
|
||||||
|
t.Fatalf("validation wrote success output: %q", stdout)
|
||||||
|
}
|
||||||
|
if mode == "show" && (!strings.Contains(stdout, "profiles:") || !strings.Contains(stdout, "offline-model")) {
|
||||||
|
t.Fatalf("effective framework YAML is missing: %q", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameworkConfigurationSourcesProjectsOrderedNativeRecords(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
importDir := filepath.Join(root, "catalog")
|
||||||
|
if err := os.MkdirAll(importDir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mainPath := writeFrameworkSourceFile(t, root, "main.yml", `configkit:
|
||||||
|
imports:
|
||||||
|
- path: catalog/fragment.yml
|
||||||
|
backends:
|
||||||
|
local:
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
profiles:
|
||||||
|
selected:
|
||||||
|
backend: local
|
||||||
|
model: selected-model
|
||||||
|
`)
|
||||||
|
fragmentPath := writeFrameworkSourceFile(t, importDir, "fragment.yml", `profiles:
|
||||||
|
imported:
|
||||||
|
model: imported-model
|
||||||
|
`)
|
||||||
|
directory := t.TempDir()
|
||||||
|
directoryPath := writeFrameworkSourceFile(t, directory, "directory.yml", `profiles:
|
||||||
|
directory:
|
||||||
|
model: directory-model
|
||||||
|
`)
|
||||||
|
configuredMissing := filepath.Join(root, "configured-but-replaced.yml")
|
||||||
|
configPath := writeAppConfigFile(t, fmt.Sprintf("promptkit_sources:\n - file: %q\n", configuredMissing))
|
||||||
|
args := []string{
|
||||||
|
"configuration", "sources", "--config", configPath,
|
||||||
|
"--promptkit-source", mainPath,
|
||||||
|
"--promptkit-source", directory,
|
||||||
|
}
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||||
|
if code != ExitOK || stderr != "" || !strings.HasSuffix(stdout, "\n") {
|
||||||
|
t.Fatalf("framework sources command failed: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
var sources []frameworkConfigurationSource
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &sources); err != nil {
|
||||||
|
t.Fatalf("decode framework source DTOs: %v\n%s", err, stdout)
|
||||||
|
}
|
||||||
|
var rawSources []map[string]any
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &rawSources); err != nil {
|
||||||
|
t.Fatalf("decode framework source fields: %v", err)
|
||||||
|
}
|
||||||
|
mainIndex := findFrameworkSource(t, sources, mainPath)
|
||||||
|
fragmentIndex := findFrameworkSource(t, sources, fragmentPath)
|
||||||
|
directoryIndex := findFrameworkSource(t, sources, directoryPath)
|
||||||
|
main := sources[mainIndex]
|
||||||
|
fragment := sources[fragmentIndex]
|
||||||
|
directorySource := sources[directoryIndex]
|
||||||
|
if !(mainIndex < fragmentIndex && fragmentIndex < directoryIndex) {
|
||||||
|
t.Fatalf("captured source order changed: main=%d fragment=%d directory=%d", mainIndex, fragmentIndex, directoryIndex)
|
||||||
|
}
|
||||||
|
if main.ID == "" || main.Label == "" || main.Layer != configkit.LayerUnderlay || main.Kind != configkit.SourceKindFile || main.UnderlayIndex < 1 {
|
||||||
|
t.Fatalf("explicit file source fields are incomplete: %+v", main)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"id", "label", "layer", "kind", "underlay_index", "path"} {
|
||||||
|
if _, exists := rawSources[mainIndex][key]; !exists {
|
||||||
|
t.Fatalf("source DTO omitted %q: %#v", key, rawSources[mainIndex])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range []string{"ID", "Label", "Layer", "Kind", "UnderlayIndex", "filesystem"} {
|
||||||
|
if _, exists := rawSources[mainIndex][key]; exists {
|
||||||
|
t.Fatalf("source DTO exposed implementation field %q: %#v", key, rawSources[mainIndex])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fragment.Layer != configkit.LayerUnderlay || fragment.Kind != configkit.SourceKindFile || fragment.ImportedFrom == nil ||
|
||||||
|
fragment.ImportedFrom.SourceID != main.ID || fragment.ImportedFrom.Line < 1 || fragment.ImportedFrom.Column < 1 {
|
||||||
|
t.Fatalf("import source fields are incomplete: %+v", fragment)
|
||||||
|
}
|
||||||
|
if importedFrom, ok := rawSources[fragmentIndex]["imported_from"].(map[string]any); !ok || importedFrom["source_id"] != main.ID || importedFrom["line"] == nil || importedFrom["column"] == nil {
|
||||||
|
t.Fatalf("import location DTO fields are incomplete: %#v", rawSources[fragmentIndex])
|
||||||
|
}
|
||||||
|
if directorySource.Layer != configkit.LayerUnderlay || directorySource.Kind != configkit.SourceKindFile || directorySource.UnderlayIndex <= main.UnderlayIndex {
|
||||||
|
t.Fatalf("directory-selected source fields/order are incomplete: %+v (main %+v)", directorySource, main)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameworkConfigurationShowFailsBeforeOutputForInvalidFinalDefinition(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
frameworkFile := writeFrameworkSourceFile(t, root, "invalid.yml", `prompts:
|
||||||
|
unused:
|
||||||
|
"1":
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "{{"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`)
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, []string{
|
||||||
|
"configuration", "show", "--promptkit-source", frameworkFile,
|
||||||
|
})
|
||||||
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||||
|
t.Fatalf("invalid final framework definition was reported as success: code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFrameworkConfigurationCommandsRejectExtraArgumentsAndReportWriterErrors(t *testing.T) {
|
||||||
|
configPath := writeAppConfigFile(t, "{}\n")
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"configuration"},
|
||||||
|
{"configuration", "explain"},
|
||||||
|
{"configuration", "show", "unexpected"},
|
||||||
|
{"configuration", "sources", "--format", "json"},
|
||||||
|
} {
|
||||||
|
code, stdout, stderr := runCLICommand(t, inspectCommand, args)
|
||||||
|
if code != ExitRuntimeError || stdout != "" || stderr == "" {
|
||||||
|
t.Fatalf("invalid inspection arguments were accepted: args=%q code=%d stdout=%q stderr=%q", args, code, stdout, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writerError := errors.New("synthetic framework output failure")
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := inspectCommand([]string{"configuration", "show", "--config", configPath}, frameworkConfigurationFailingWriter{err: writerError}, &stderr)
|
||||||
|
if code != ExitRuntimeError || !strings.Contains(stderr.String(), writerError.Error()) {
|
||||||
|
t.Fatalf("effective-output writer failure was not reported: code=%d stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findFrameworkSource(t *testing.T, sources []frameworkConfigurationSource, path string) int {
|
||||||
|
t.Helper()
|
||||||
|
for index, source := range sources {
|
||||||
|
if filepath.Clean(source.Path) == filepath.Clean(path) {
|
||||||
|
return index
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("selected source path %q was not present: %+v", path, sources)
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
type frameworkConfigurationFailingWriter struct {
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (writer frameworkConfigurationFailingWriter) Write([]byte) (int, error) {
|
||||||
|
return 0, writer.err
|
||||||
|
}
|
||||||
@@ -39,6 +39,9 @@ func inspectCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
if args[0] == "profile" {
|
if args[0] == "profile" {
|
||||||
return inspectProfileCommand(args[1:], stdout, stderr)
|
return inspectProfileCommand(args[1:], stdout, stderr)
|
||||||
}
|
}
|
||||||
|
if args[0] == "configuration" {
|
||||||
|
return inspectFrameworkConfigurationCommand(args[1:], stdout, stderr)
|
||||||
|
}
|
||||||
if args[0] != "prompt" {
|
if args[0] != "prompt" {
|
||||||
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
|
fmt.Fprintln(stderr, "inspect parse error: unknown inspection mode")
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
|
|||||||
Reference in New Issue
Block a user