From bb63c2cb101f3b68c61ae6c42d5a5a6d8b828840 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 10 Nov 2026 20:35:35 +0000 Subject: [PATCH] Collect prompt file paths from effective configuration --- docs/roadmap/remediation.md | 2 + internal/cli/app_wiring.go | 6 +- internal/framework/llm/runtime.go | 23 +++-- .../framework/llm/runtime_assembly_test.go | 12 +-- .../framework/llm/runtime_inspection_test.go | 2 +- internal/framework/llm/source_paths.go | 89 ++++++++++--------- internal/framework/llm/source_paths_test.go | 21 +++-- 7 files changed, 87 insertions(+), 68 deletions(-) diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 4b9e32c..30a19ea 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -451,6 +451,8 @@ selected vocabulary. ## R13 — P2: Collect Prompt Message Paths From Effective Data +**Status:** Completed. Processing walks one effective PromptKit projection and uses per-field origins for host message paths. Inspection and request commands skip collision accounting. Tests cover inherited, collection-relative, and symlinked message sources. + **Category** Repeated configuration history replay in Stages 3, 5, and 6. diff --git a/internal/cli/app_wiring.go b/internal/cli/app_wiring.go index ebfe522..03cc56f 100644 --- a/internal/cli/app_wiring.go +++ b/internal/cli/app_wiring.go @@ -52,11 +52,7 @@ func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input s if state != nil { client = state.client } - runtime, err := llm.NewRuntime(ctx, loaded, client) - if err != nil { - return nil, err - } - return runtime.Engine, nil + return llm.NewEngine(ctx, loaded, client) } func capturePromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) { diff --git a/internal/framework/llm/runtime.go b/internal/framework/llm/runtime.go index d97efc1..b63da72 100644 --- a/internal/framework/llm/runtime.go +++ b/internal/framework/llm/runtime.go @@ -41,10 +41,23 @@ const operatorImports = `config: ` func NewRuntime(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client) (*Runtime, error) { - return newRuntimeWithPromptFS(ctx, loaded, client, prompts.Filesystem()) + engine, err := NewEngine(ctx, loaded, client) + if err != nil { + return nil, err + } + paths, err := capturedSourcePaths(engine) + if err != nil { + return nil, fmt.Errorf("inspect captured LLM sources: %w", err) + } + return &Runtime{Config: loaded.Config, Engine: engine, SourcePaths: paths}, nil } -func newRuntimeWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client, promptFS fs.FS) (*Runtime, error) { +// NewEngine loads the captured PromptKit resources without processing collision checks. +func NewEngine(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client) (*pk.Engine, error) { + return newEngineWithPromptFS(ctx, loaded, client, prompts.Filesystem()) +} + +func newEngineWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client, promptFS fs.FS) (*pk.Engine, error) { if err := ctx.Err(); err != nil { return nil, err } @@ -93,11 +106,7 @@ func newRuntimeWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config return nil, embeddedAssetOrInputError(ctx, embedded, fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)) } } - paths, err := capturedSourcePaths(engine) - if err != nil { - return nil, fmt.Errorf("inspect captured LLM sources: %w", err) - } - return &Runtime{Config: loaded.Config, Engine: engine, SourcePaths: paths}, nil + return engine, nil } func embeddedAssetOrInputError(ctx context.Context, inputs []ck.Input, cause error) error { diff --git a/internal/framework/llm/runtime_assembly_test.go b/internal/framework/llm/runtime_assembly_test.go index 73030b1..ecf4c93 100644 --- a/internal/framework/llm/runtime_assembly_test.go +++ b/internal/framework/llm/runtime_assembly_test.go @@ -41,13 +41,13 @@ func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{}) + _, err = newEngineWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{}) if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) { t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err) } ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = newRuntimeWithPromptFS(ctx, loaded, nil, fstest.MapFS{}) + _, err = newEngineWithPromptFS(ctx, loaded, nil, fstest.MapFS{}) if !errors.Is(err, context.Canceled) || IsAssetError(err) { t.Fatalf("caller cancellation classification: %v", err) } @@ -61,7 +61,7 @@ func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS) + _, err = newEngineWithPromptFS(context.Background(), loaded, nil, promptFS) if err == nil || !IsAssetError(err) { t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err) } @@ -84,11 +84,11 @@ func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) { if err != nil { t.Fatal(err) } - runtime, err := newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS) + engine, err := newEngineWithPromptFS(context.Background(), loaded, nil, promptFS) if err != nil { t.Fatalf("valid final configuration rejected: %v", err) } - if runtime.Engine == nil { - t.Fatal("runtime engine was not retained") + if engine == nil { + t.Fatal("engine was not retained") } } diff --git a/internal/framework/llm/runtime_inspection_test.go b/internal/framework/llm/runtime_inspection_test.go index 1cf7750..78901d8 100644 --- a/internal/framework/llm/runtime_inspection_test.go +++ b/internal/framework/llm/runtime_inspection_test.go @@ -24,7 +24,7 @@ func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil { + if _, err := NewEngine(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil { t.Fatal(err) } } diff --git a/internal/framework/llm/source_paths.go b/internal/framework/llm/source_paths.go index 84f7d70..b477764 100644 --- a/internal/framework/llm/source_paths.go +++ b/internal/framework/llm/source_paths.go @@ -1,11 +1,10 @@ package llm import ( + "bytes" "fmt" "path/filepath" "sort" - "strconv" - "strings" ck "go.maximumdirect.net/configkit" pk "go.maximumdirect.net/promptkit" @@ -19,54 +18,58 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) { } sources := inspection.Sources() - byID := make(map[string]ck.SourceInfo, len(sources)) + hostByID := make(map[string]string, len(sources)) paths := make(map[string]struct{}, len(sources)) - hostPath := func(source ck.SourceInfo) string { - if source.Kind == ck.SourceKindFile { - return source.Path - } - return "" - } for _, source := range sources { - byID[source.ID] = source - if path := hostPath(source); path != "" { - paths[path] = struct{}{} + if source.Kind == ck.SourceKindFile && source.Path != "" { + hostByID[source.ID] = source.Path + paths[source.Path] = struct{}{} } } + if len(hostByID) == 0 { + return []string{}, nil + } + + type promptList []struct { + Messages []struct { + ContentFile *string `yaml:"content_file"` + } `yaml:"messages"` + } + var effective struct { + Prompts promptList `yaml:"prompts"` + Collections map[string]struct { + Prompts promptList `yaml:"prompts"` + } `yaml:"collections"` + } + var projection bytes.Buffer + if err := inspection.WriteEffective(&projection); err != nil { + return nil, fmt.Errorf("write effective PromptKit configuration: %w", err) + } + if err := yaml.Unmarshal(projection.Bytes(), &effective); err != nil { + return nil, fmt.Errorf("decode effective PromptKit message paths: %w", err) + } metadata := inspection.Metadata() - for _, path := range metadata.HistoricalPaths() { - segments := strings.Split(strings.TrimPrefix(path.String(), "/"), "/") - // Prompt resources are lists at the root or inside a collection. - if len(segments) == 7 && segments[0] == "collections" { - segments = segments[2:] + capture := func(base ck.Path, prompts promptList) { + for i, prompt := range prompts { + messages := base.Index(i).Key("messages") + for j, message := range prompt.Messages { + if message.ContentFile == nil { + continue + } + field, ok := metadata.Lookup(messages.Index(j).Key("content_file")) + if !ok || len(field.Origin.Locations) != 1 { + continue + } + if host := hostByID[field.Origin.Locations[0].SourceID]; host != "" { + paths[filepath.Join(filepath.Dir(host), filepath.FromSlash(*message.ContentFile))] = struct{}{} + } + } } - if len(segments) != 5 || segments[0] != "prompts" || segments[2] != "messages" || segments[4] != "content_file" { - continue - } - if _, err := strconv.Atoi(segments[1]); err != nil { - continue - } - if _, err := strconv.Atoi(segments[3]); err != nil { - continue - } - field, ok := metadata.Lookup(path) - if !ok || field.Null || len(field.Origin.Locations) != 1 { - continue - } - source, ok := byID[field.Origin.Locations[0].SourceID] - if !ok || hostPath(source) == "" { - continue - } - explanation, ok := inspection.Explain(path) - if !ok || !explanation.Effective.Present || explanation.Effective.Redacted { - continue - } - var resource string - if err := yaml.Unmarshal([]byte(explanation.Effective.YAML), &resource); err != nil { - return nil, fmt.Errorf("decode captured message resource at %s: %w", path.String(), err) - } - paths[filepath.Join(filepath.Dir(hostPath(source)), filepath.FromSlash(resource))] = struct{}{} + } + capture(ck.Keys("prompts"), effective.Prompts) + for name, collection := range effective.Collections { + capture(ck.Keys("collections", name, "prompts"), collection.Prompts) } result := make([]string, 0, len(paths)) diff --git a/internal/framework/llm/source_paths_test.go b/internal/framework/llm/source_paths_test.go index 9065544..0516555 100644 --- a/internal/framework/llm/source_paths_test.go +++ b/internal/framework/llm/source_paths_test.go @@ -15,11 +15,13 @@ import ( func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) { dir := t.TempDir() for name, content := range map[string]string{ - "config.yml": "version: 3\n", - "prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n", - "catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n", - "catalog/instruction.md": "Captured operator message.\n", - "prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n", + "config.yml": "version: 3\n", + "prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n", + "catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n", + "catalog/instruction.md": "Captured operator message.\n", + "prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n", + "prompts/nested/collection.yml": "collections:\n operator:\n prompts:\n - id: local\n messages:\n - role: user\n content_file: ../shared/collection.md\n output:\n format: text\n validation_mode: none\n", + "prompts/shared/collection.md": "Message in a collection.\n", } { file := filepath.Join(dir, name) if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { @@ -37,7 +39,14 @@ func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T if err != nil { t.Fatal(err) } - want := []string{filepath.Join(dir, "prompts.yml"), filepath.Join(dir, "catalog/arbitrary-name.yaml"), filepath.Join(dir, "catalog/instruction.md"), filepath.Join(dir, "prompts/nested/override.yaml")} + want := []string{ + filepath.Join(dir, "prompts.yml"), + filepath.Join(dir, "catalog/arbitrary-name.yaml"), + filepath.Join(dir, "catalog/instruction.md"), + filepath.Join(dir, "prompts/nested/override.yaml"), + filepath.Join(dir, "prompts/nested/collection.yml"), + filepath.Join(dir, "prompts/shared/collection.md"), + } sort.Strings(want) if !reflect.DeepEqual(runtime.SourcePaths, want) { t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want)