eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Collect prompt file paths from effective configuration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 20:35:35 +00:00
parent 7710101da0
commit bb63c2cb10
7 changed files with 87 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/cli/app_wiring.go
View File

@@ -52,11 +52,7 @@ func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input s
if state != nil {
client = state.client
}
runtime, err := llm.NewRuntime(ctx, loaded, client)
if err != nil {
return nil, err
}
return runtime.Engine, nil
return llm.NewEngine(ctx, loaded, client)
}
func capturePromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {

23
internal/framework/llm/runtime.go
View File

@@ -41,10 +41,23 @@ const operatorImports = `config:
`
func NewRuntime(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client) (*Runtime, error) {
return newRuntimeWithPromptFS(ctx, loaded, client, prompts.Filesystem())
engine, err := NewEngine(ctx, loaded, client)
if err != nil {
return nil, err
}
paths, err := capturedSourcePaths(engine)
if err != nil {
return nil, fmt.Errorf("inspect captured LLM sources: %w", err)
}
return &Runtime{Config: loaded.Config, Engine: engine, SourcePaths: paths}, nil
}
func newRuntimeWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client, promptFS fs.FS) (*Runtime, error) {
// NewEngine loads the captured PromptKit resources without processing collision checks.
func NewEngine(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client) (*pk.Engine, error) {
return newEngineWithPromptFS(ctx, loaded, client, prompts.Filesystem())
}
func newEngineWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client, promptFS fs.FS) (*pk.Engine, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
@@ -93,11 +106,7 @@ func newRuntimeWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config
return nil, embeddedAssetOrInputError(ctx, embedded, fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err))
}
}
paths, err := capturedSourcePaths(engine)
if err != nil {
return nil, fmt.Errorf("inspect captured LLM sources: %w", err)
}
return &Runtime{Config: loaded.Config, Engine: engine, SourcePaths: paths}, nil
return engine, nil
}
func embeddedAssetOrInputError(ctx context.Context, inputs []ck.Input, cause error) error {

12
internal/framework/llm/runtime_assembly_test.go
View File

@@ -41,13 +41,13 @@ func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{})
_, err = newEngineWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{})
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = newRuntimeWithPromptFS(ctx, loaded, nil, fstest.MapFS{})
_, err = newEngineWithPromptFS(ctx, loaded, nil, fstest.MapFS{})
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
t.Fatalf("caller cancellation classification: %v", err)
}
@@ -61,7 +61,7 @@ func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
_, err = newEngineWithPromptFS(context.Background(), loaded, nil, promptFS)
if err == nil || !IsAssetError(err) {
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
}
@@ -84,11 +84,11 @@ func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
if err != nil {
t.Fatal(err)
}
runtime, err := newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
engine, err := newEngineWithPromptFS(context.Background(), loaded, nil, promptFS)
if err != nil {
t.Fatalf("valid final configuration rejected: %v", err)
}
if runtime.Engine == nil {
t.Fatal("runtime engine was not retained")
if engine == nil {
t.Fatal("engine was not retained")
}
}

2
internal/framework/llm/runtime_inspection_test.go
View File

@@ -24,7 +24,7 @@ func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil {
if _, err := NewEngine(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
}

89
internal/framework/llm/source_paths.go
View File

@@ -1,11 +1,10 @@
package llm
import (
"bytes"
"fmt"
"path/filepath"
"sort"
"strconv"
"strings"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
@@ -19,54 +18,58 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) {
}
sources := inspection.Sources()
byID := make(map[string]ck.SourceInfo, len(sources))
hostByID := make(map[string]string, len(sources))
paths := make(map[string]struct{}, len(sources))
hostPath := func(source ck.SourceInfo) string {
if source.Kind == ck.SourceKindFile {
return source.Path
}
return ""
}
for _, source := range sources {
byID[source.ID] = source
if path := hostPath(source); path != "" {
paths[path] = struct{}{}
if source.Kind == ck.SourceKindFile && source.Path != "" {
hostByID[source.ID] = source.Path
paths[source.Path] = struct{}{}
}
}
if len(hostByID) == 0 {
return []string{}, nil
}
type promptList []struct {
Messages []struct {
ContentFile *string `yaml:"content_file"`
} `yaml:"messages"`
}
var effective struct {
Prompts promptList `yaml:"prompts"`
Collections map[string]struct {
Prompts promptList `yaml:"prompts"`
} `yaml:"collections"`
}
var projection bytes.Buffer
if err := inspection.WriteEffective(&projection); err != nil {
return nil, fmt.Errorf("write effective PromptKit configuration: %w", err)
}
if err := yaml.Unmarshal(projection.Bytes(), &effective); err != nil {
return nil, fmt.Errorf("decode effective PromptKit message paths: %w", err)
}
metadata := inspection.Metadata()
for _, path := range metadata.HistoricalPaths() {
segments := strings.Split(strings.TrimPrefix(path.String(), "/"), "/")
// Prompt resources are lists at the root or inside a collection.
if len(segments) == 7 && segments[0] == "collections" {
segments = segments[2:]
capture := func(base ck.Path, prompts promptList) {
for i, prompt := range prompts {
messages := base.Index(i).Key("messages")
for j, message := range prompt.Messages {
if message.ContentFile == nil {
continue
}
field, ok := metadata.Lookup(messages.Index(j).Key("content_file"))
if !ok || len(field.Origin.Locations) != 1 {
continue
}
if host := hostByID[field.Origin.Locations[0].SourceID]; host != "" {
paths[filepath.Join(filepath.Dir(host), filepath.FromSlash(*message.ContentFile))] = struct{}{}
}
}
}
if len(segments) != 5 || segments[0] != "prompts" || segments[2] != "messages" || segments[4] != "content_file" {
continue
}
if _, err := strconv.Atoi(segments[1]); err != nil {
continue
}
if _, err := strconv.Atoi(segments[3]); err != nil {
continue
}
field, ok := metadata.Lookup(path)
if !ok || field.Null || len(field.Origin.Locations) != 1 {
continue
}
source, ok := byID[field.Origin.Locations[0].SourceID]
if !ok || hostPath(source) == "" {
continue
}
explanation, ok := inspection.Explain(path)
if !ok || !explanation.Effective.Present || explanation.Effective.Redacted {
continue
}
var resource string
if err := yaml.Unmarshal([]byte(explanation.Effective.YAML), &resource); err != nil {
return nil, fmt.Errorf("decode captured message resource at %s: %w", path.String(), err)
}
paths[filepath.Join(filepath.Dir(hostPath(source)), filepath.FromSlash(resource))] = struct{}{}
}
capture(ck.Keys("prompts"), effective.Prompts)
for name, collection := range effective.Collections {
capture(ck.Keys("collections", name, "prompts"), collection.Prompts)
}
result := make([]string, 0, len(paths))

21
internal/framework/llm/source_paths_test.go
View File

@@ -15,11 +15,13 @@ import (
func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n",
"catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n",
"catalog/instruction.md": "Captured operator message.\n",
"prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n",
"config.yml": "version: 3\n",
"prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n",
"catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n",
"catalog/instruction.md": "Captured operator message.\n",
"prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n",
"prompts/nested/collection.yml": "collections:\n operator:\n prompts:\n - id: local\n messages:\n - role: user\n content_file: ../shared/collection.md\n output:\n format: text\n validation_mode: none\n",
"prompts/shared/collection.md": "Message in a collection.\n",
} {
file := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
@@ -37,7 +39,14 @@ func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T
if err != nil {
t.Fatal(err)
}
want := []string{filepath.Join(dir, "prompts.yml"), filepath.Join(dir, "catalog/arbitrary-name.yaml"), filepath.Join(dir, "catalog/instruction.md"), filepath.Join(dir, "prompts/nested/override.yaml")}
want := []string{
filepath.Join(dir, "prompts.yml"),
filepath.Join(dir, "catalog/arbitrary-name.yaml"),
filepath.Join(dir, "catalog/instruction.md"),
filepath.Join(dir, "prompts/nested/override.yaml"),
filepath.Join(dir, "prompts/nested/collection.yml"),
filepath.Join(dir, "prompts/shared/collection.md"),
}
sort.Strings(want)
if !reflect.DeepEqual(runtime.SourcePaths, want) {
t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want)