Load bundled and operator prompt resources

2026-11-10 16:13:53 +00:00
parent 7c64592cab
commit 497a1a18c5
13 changed files with 502 additions and 353 deletions
--- a/internal/framework/llm/engine_sources_test.go
+++ b/internal/framework/llm/engine_sources_test.go
@@ -0,0 +1,163 @@
+package llm
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
+	"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
+	ck "go.maximumdirect.net/configkit"
+)
+
+func TestEngineCapturesAutomaticPromptSources(t *testing.T) {
+	dir := t.TempDir()
+	main := filepath.Join(dir, "config.yml")
+	write := func(name, text string) {
+		t.Helper()
+		if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(text), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("config.yml", "version: 3\n")
+	write("prompts.yml", "collections:\n  audita:\n    prompts:\n      - id: modules.grammar.proposal\n        messages:\n          - role: user\n            content_file: message.txt\n        output:\n          format: text\n          validation_mode: none\n")
+	write("message.txt", "first")
+	write("prompts/nested/extra.yml", "prompts:\n  - id: extra\n    messages:\n      - role: user\n        content: extra\n    output:\n      format: text\n      validation_mode: none\n")
+	loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	first, err := NewRuntime(context.Background(), loaded, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	inspect := func(r *Runtime) string {
+		t.Helper()
+		p, err := r.Engine.InspectPrompt(context.Background(), prompts.PromptIDModuleGrammarProposal)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return p.Messages[0].Content
+	}
+	if got := inspect(first); got != "first" {
+		t.Fatalf("prompt = %q", got)
+	}
+	for _, name := range []string{"prompts.yml", "message.txt", "prompts/nested/extra.yml"} {
+		if !slices.Contains(first.SourcePaths, filepath.Join(dir, name)) {
+			t.Errorf("missing %s in %v", name, first.SourcePaths)
+		}
+	}
+	if _, err := first.Engine.InspectPrompt(context.Background(), prompts.PromptIDModuleGlossaryProposal); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := first.Engine.InspectPrompt(context.Background(), "extra"); err != nil {
+		t.Fatal(err)
+	}
+	write("message.txt", "second")
+	second, err := NewRuntime(context.Background(), loaded, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if inspect(first) != "first" || inspect(second) != "second" {
+		t.Fatal("engine did not capture source at construction")
+	}
+}
+
+func TestEngineWithoutMainUsesEmbeddedPrompts(t *testing.T) {
+	loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	runtime, err := NewRuntime(context.Background(), loaded, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(runtime.SourcePaths) != 0 {
+		t.Fatalf("unexpected paths: %v", runtime.SourcePaths)
+	}
+	if _, err := runtime.Engine.InspectPrompt(context.Background(), prompts.PromptIDModuleGrammarProposal); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEngineCapturesInheritedMessageAndRejectsPeerConflicts(t *testing.T) {
+	dir := t.TempDir()
+	write := func(name, text string) {
+		t.Helper()
+		file := filepath.Join(dir, name)
+		if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(file, []byte(text), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("config.yml", "version: 3\n")
+	write("prompts.yml", "config:\n  imports:\n    - path: shared/catalog.yml\n      required: true\n")
+	write("shared/catalog.yml", "prompts:\n  - id: parent\n    messages:\n      - role: user\n        content_file: instruction.md\n    output:\n      format: text\n      validation_mode: none\n")
+	write("shared/instruction.md", "inherited content")
+	write("prompts/child.yml", "prompts:\n  - id: child\n    extends: {id: parent}\n")
+	loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	runtime, err := NewRuntime(context.Background(), loaded, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	prompt, err := runtime.Engine.InspectPrompt(context.Background(), "child")
+	if err != nil || prompt.Messages[0].Content != "inherited content" {
+		t.Fatalf("inherited prompt: %+v, %v", prompt, err)
+	}
+	for _, name := range []string{"prompts.yml", "shared/catalog.yml", "shared/instruction.md", "prompts/child.yml"} {
+		if !slices.Contains(runtime.SourcePaths, filepath.Join(dir, name)) {
+			t.Errorf("missing %s in %v", name, runtime.SourcePaths)
+		}
+	}
+	write("prompts/duplicate.yml", "prompts:\n  - id: parent\n    messages:\n      - role: user\n        content: duplicate\n    output:\n      format: text\n      validation_mode: none\n")
+	if _, err := NewRuntime(context.Background(), loaded, nil); err == nil || IsAssetError(err) {
+		t.Fatalf("peer conflict classification: %v", err)
+	}
+}
+
+func TestEngineUsesRecordedContextWithWiderRoot(t *testing.T) {
+	root := t.TempDir()
+	project := filepath.Join(root, "campaign")
+	if err := os.Mkdir(project, 0755); err != nil {
+		t.Fatal(err)
+	}
+	for name, body := range map[string]string{
+		"config.yml":  "version: 3\n",
+		"prompts.yml": "config:\n  imports:\n    - path: ../shared.yml\n      required: true\n",
+	} {
+		if err := os.WriteFile(filepath.Join(project, name), []byte(body), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(root, "shared.yml"), []byte("prompts:\n  - id: shared\n    messages:\n      - role: user\n        content: shared\n    output:\n      format: text\n      validation_mode: none\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	source := ck.SearchConfig("audita", ck.SearchDirectory(project))
+	loaded, err := config.Load(ck.Input{Source: source, Filesystem: &ck.FilesystemContext{Root: root}}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Chdir(t.TempDir())
+	runtime, err := NewRuntime(context.Background(), loaded, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := runtime.Engine.InspectPrompt(context.Background(), "shared"); err != nil {
+		t.Fatal(err)
+	}
+	for _, file := range []string{filepath.Join(project, "prompts.yml"), filepath.Join(root, "shared.yml")} {
+		if !slices.Contains(runtime.SourcePaths, file) {
+			t.Errorf("missing %s in %v", file, runtime.SourcePaths)
+		}
+	}
+}
--- a/internal/framework/llm/runtime.go
+++ b/internal/framework/llm/runtime.go
@@ -10,147 +10,108 @@ import (
 	"strings"
 	"sync/atomic"

-	ck "gitea.maximumdirect.net/eric/ConfigKit"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
-	pk "gitea.maximumdirect.net/eric/promptkit"
+	ck "go.maximumdirect.net/configkit"
+	pk "go.maximumdirect.net/promptkit"
 )

-// Runtime owns one PromptKit engine and the validated selections for both roles.
-// The same engine is shared by the role-bound clients during processing.
+// Runtime owns the captured engine shared by all processing clients.
 type Runtime struct {
-	Config   config.Config
-	Engine   *pk.Engine
-	Profiles map[string]pk.ProfileInspection
-	// SourcePaths contains host configuration and message files captured by PromptKit.
+	Config      config.Config
+	Engine      *pk.Engine
 	SourcePaths []string
 	nextCallID  atomic.Uint64
 }

-// AssetError indicates a broken embedded prompt or schema rather than an
-// operator-provided setting or profile.
 type AssetError struct{ Err error }

 func (e *AssetError) Error() string { return e.Err.Error() }
 func (e *AssetError) Unwrap() error { return e.Err }

-func NewRuntime(ctx context.Context, c config.Config, mainPath string, client *http.Client) (*Runtime, error) {
-	return newRuntimeWithPromptFS(ctx, c, mainPath, client, prompts.Filesystem())
+const operatorFilesystem = "Audita operator prompts"
+const operatorImports = `config:
+  imports:
+    - path: prompts.yml
+      required: false
+    - dir: prompts
+      required: false
+      recursive: true
+      pattern: ["*.yml", "*.yaml"]
+`
+
+func NewRuntime(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client) (*Runtime, error) {
+	return newRuntimeWithPromptFS(ctx, loaded, client, prompts.Filesystem())
 }

-func newRuntimeWithPromptFS(ctx context.Context, c config.Config, mainPath string, client *http.Client, promptFS fs.FS) (*Runtime, error) {
+func newRuntimeWithPromptFS(ctx context.Context, loaded *ck.Result[config.Config], client *http.Client, promptFS fs.FS) (*Runtime, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
 	}
-	profileDir := c.ProfilesDir
-	if !filepath.IsAbs(profileDir) {
-		profileDir = filepath.Join(filepath.Dir(mainPath), profileDir)
+	if loaded == nil {
+		return nil, fmt.Errorf("application configuration is required")
 	}
-	promptInputs, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
+	embedded, err := pk.FilesystemInputs("Audita embedded prompts", promptFS, prompts.AssetRoot)
 	if err != nil {
-		if ctxErr := ctx.Err(); ctxErr != nil {
-			return nil, ctxErr
-		}
 		return nil, &AssetError{Err: fmt.Errorf("discover embedded prompts: %w", err)}
 	}
-	profileInputs, err := pk.DirectoryInputs(profileDir)
-	if err != nil {
-		if ctxErr := ctx.Err(); ctxErr != nil {
-			return nil, ctxErr
+	sources := pk.ConfigurationSources{Underlays: []ck.Underlay{{Inputs: embedded}}}
+	var hostRoot string
+	for _, source := range loaded.Sources() {
+		if source.Layer != ck.LayerMain || source.ImportedFrom != nil {
+			continue
 		}
-		return nil, fmt.Errorf("discover operator definitions: %w", err)
-	}
-	if err := ctx.Err(); err != nil {
-		return nil, err
-	}
-	applicationBackends := make(map[string]any, len(c.Backends))
-	for id := range c.Backends {
-		backend := c.Backends[id]
-		applicationBackends[id] = map[string]any{
-			"endpoint":          backend.Endpoint,
-			"concurrency_limit": backend.Concurrency,
-			"queue_capacity":    backend.QueueCapacity,
-			"api_key_env":       nil,
+		context, ok := loaded.SourceContext(source.ID)
+		if !ok {
+			break
 		}
+		filesystem := ck.NewFilesystem(operatorFilesystem, context.FS)
+		sources.Underlays = append(sources.Underlays, ck.Underlay{Inputs: []ck.Input{{
+			Source: ck.Bytes("Audita automatic prompt imports", []byte(operatorImports)), Imports: true,
+			Filesystem: &ck.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: context.BaseDir},
+		}}})
+		if source.Kind == ck.SourceKindFile && source.Path != "" {
+			hostRoot = filepath.Dir(source.Path)
+			if context.BaseDir != "." {
+				for range strings.Split(context.BaseDir, "/") {
+					hostRoot = filepath.Dir(hostRoot)
+				}
+			}
+		}
+		break
 	}
-	sources := pk.ConfigurationSources{
-		Underlays: []ck.Underlay{
-			{Inputs: promptInputs},
-			{Inputs: profileInputs},
-		},
-		Main: &ck.Input{Source: ck.Mapping("application backends", map[string]any{"backends": applicationBackends})},
-	}
-	engine, err := pk.NewEngine(pk.Config{
-		Sources:          sources,
-		HTTPClient:       client,
-		TransientRetries: &c.LLM.TransientRetries,
-	}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
+	engine, err := pk.NewEngine(pk.Config{Sources: sources, HTTPClient: client}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
 	if err != nil {
-		return nil, embeddedAssetOrInputError(ctx, promptInputs, fmt.Errorf("assemble LLM runtime: %w", err))
+		return nil, embeddedAssetOrInputError(ctx, embedded, fmt.Errorf("assemble LLM runtime: %w", err))
 	}
 	for _, meta := range prompts.RegisteredMetadata() {
-		if _, err := engine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); err != nil {
-			cause := fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err)
-			return nil, embeddedAssetOrInputError(ctx, promptInputs, cause)
+		if _, err := engine.InspectPrompt(ctx, meta.PromptID); err != nil {
+			return nil, embeddedAssetOrInputError(ctx, embedded, fmt.Errorf("inspect prompt %s: %w", meta.PromptID, err))
 		}
 	}
-	runtime := &Runtime{Config: c, Engine: engine, Profiles: make(map[string]pk.ProfileInspection, 2)}
-	for _, selection := range []struct{ role, id string }{{"proposal", c.LLM.ProposalProfile}, {"validation", c.LLM.ValidationProfile}} {
-		profile, err := engine.InspectProfile(ctx, selection.id)
-		if err != nil {
-			return nil, fmt.Errorf("inspect %s profile: %w", selection.role, err)
-		}
-		target := profile.EffectiveModelParams
-		backend, ok := c.Backends[target.BackendID]
-		if !ok {
-			return nil, fmt.Errorf("%s profile must select a registered application backend", selection.role)
-		}
-		if strings.TrimSpace(target.Model) == "" {
-			return nil, fmt.Errorf("%s profile must select a nonblank model", selection.role)
-		}
-		if strings.TrimSpace(target.Endpoint) == "" {
-			return nil, fmt.Errorf("%s profile must resolve to a nonblank endpoint", selection.role)
-		}
-		endpoint := func(value string) string { return strings.TrimRight(strings.TrimSpace(value), "/") }
-		if endpoint(target.Endpoint) != endpoint(backend.Endpoint) {
-			return nil, fmt.Errorf("%s profile endpoint must match its registered backend", selection.role)
-		}
-		if target.APIKeyEnv != "" {
-			return nil, fmt.Errorf("%s profile must use the backend credential, not its own environment lookup", selection.role)
-		}
-		if profile.APIKeyRequired && backend.Credential == nil {
-			return nil, fmt.Errorf("%s profile requires a backend credential", selection.role)
-		}
-		runtime.Profiles[selection.role] = *profile
-	}
-	runtime.SourcePaths, err = capturedSourcePaths(engine)
+	paths, err := capturedSourcePaths(engine, hostRoot)
 	if err != nil {
 		return nil, fmt.Errorf("inspect captured LLM sources: %w", err)
 	}
-	return runtime, nil
+	return &Runtime{Config: loaded.Config, Engine: engine, SourcePaths: paths}, nil
 }

-func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cause error) error {
+func embeddedAssetOrInputError(ctx context.Context, inputs []ck.Input, cause error) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
-	embeddedSources := pk.ConfigurationSources{
-		Underlays: []ck.Underlay{{Inputs: promptInputs}},
-	}
-	diagnosticEngine, err := pk.NewEngine(pk.Config{Sources: embeddedSources},
-		pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
+	engine, err := pk.NewEngine(pk.Config{Sources: pk.ConfigurationSources{Underlays: []ck.Underlay{{Inputs: inputs}}}}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
 	if err == nil {
 		for _, meta := range prompts.RegisteredMetadata() {
-			if _, inspectErr := diagnosticEngine.InspectPrompt(ctx, meta.PromptID, meta.PromptVersion); inspectErr != nil {
-				err = fmt.Errorf("inspect embedded prompt %s: %w", meta.PromptID, inspectErr)
+			if _, err = engine.InspectPrompt(ctx, meta.PromptID); err != nil {
 				break
 			}
 		}
 	}
-	if ctxErr := ctx.Err(); ctxErr != nil {
-		return ctxErr
+	if err := ctx.Err(); err != nil {
+		return err
 	}
 	if err != nil {
 		return &AssetError{Err: fmt.Errorf("validate embedded prompt assets: %w", err)}
@@ -160,8 +121,6 @@ func embeddedAssetOrInputError(ctx context.Context, promptInputs []ck.Input, cau

 func IsAssetError(err error) bool { var target *AssetError; return errors.As(err, &target) }

-// RedactError replaces every configured resolved backend secret before a
-// library error is surfaced in CLI output or an artifact.
 func RedactError(err error, c config.Config) string {
 	if err == nil {
 		return ""
--- a/internal/framework/llm/source_paths.go
+++ b/internal/framework/llm/source_paths.go
@@ -7,12 +7,12 @@ import (
 	"strconv"
 	"strings"

-	ck "gitea.maximumdirect.net/eric/ConfigKit"
-	pk "gitea.maximumdirect.net/eric/promptkit"
+	ck "go.maximumdirect.net/configkit"
+	pk "go.maximumdirect.net/promptkit"
 	"gopkg.in/yaml.v3"
 )

-func capturedSourcePaths(engine *pk.Engine) ([]string, error) {
+func capturedSourcePaths(engine *pk.Engine, hostRoot string) ([]string, error) {
 	inspection, err := engine.InspectConfiguration()
 	if err != nil {
 		return nil, fmt.Errorf("inspect configuration: %w", err)
@@ -21,20 +21,36 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) {
 	sources := inspection.Sources()
 	byID := make(map[string]ck.SourceInfo, len(sources))
 	paths := make(map[string]struct{}, len(sources))
+	hostPath := func(source ck.SourceInfo) string {
+		if source.Kind == ck.SourceKindFile {
+			return source.Path
+		}
+		if hostRoot != "" && source.Kind == ck.SourceKindFilesystem && strings.HasPrefix(source.Label, operatorFilesystem+"/") {
+			return filepath.Join(hostRoot, filepath.FromSlash(source.Path))
+		}
+		return ""
+	}
 	for _, source := range sources {
 		byID[source.ID] = source
-		if source.Kind == ck.SourceKindFile && source.Path != "" {
-			paths[source.Path] = struct{}{}
+		if path := hostPath(source); path != "" {
+			paths[path] = struct{}{}
 		}
 	}

 	metadata := inspection.Metadata()
 	for _, path := range metadata.HistoricalPaths() {
 		segments := strings.Split(strings.TrimPrefix(path.String(), "/"), "/")
-		if len(segments) != 6 || segments[0] != "prompts" || segments[3] != "messages" || segments[5] != "content_file" {
+		// Prompt resources are lists at the root or inside a collection.
+		if len(segments) == 7 && segments[0] == "collections" {
+			segments = segments[2:]
+		}
+		if len(segments) != 5 || segments[0] != "prompts" || segments[2] != "messages" || segments[4] != "content_file" {
 			continue
 		}
-		if messageIndex, err := strconv.Atoi(segments[4]); err != nil || messageIndex < 0 {
+		if _, err := strconv.Atoi(segments[1]); err != nil {
+			continue
+		}
+		if _, err := strconv.Atoi(segments[3]); err != nil {
 			continue
 		}
 		field, ok := metadata.Lookup(path)
@@ -42,7 +58,7 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) {
 			continue
 		}
 		source, ok := byID[field.Origin.Locations[0].SourceID]
-		if !ok || source.Kind != ck.SourceKindFile || source.Path == "" {
+		if !ok || hostPath(source) == "" {
 			continue
 		}
 		explanation, ok := inspection.Explain(path)
@@ -53,7 +69,7 @@ func capturedSourcePaths(engine *pk.Engine) ([]string, error) {
 		if err := yaml.Unmarshal([]byte(explanation.Effective.YAML), &resource); err != nil {
 			return nil, fmt.Errorf("decode captured message resource at %s: %w", path.String(), err)
 		}
-		paths[filepath.Join(filepath.Dir(source.Path), filepath.FromSlash(resource))] = struct{}{}
+		paths[filepath.Join(filepath.Dir(hostPath(source)), filepath.FromSlash(resource))] = struct{}{}
 	}

 	result := make([]string, 0, len(paths))