Enable prompt preview validation and smoke commands

2026-11-10 16:59:28 +00:00
parent 8cc298d092
commit 8e1e073731
4 changed files with 184 additions and 37 deletions
--- a/internal/cli/app_wiring.go
+++ b/internal/cli/app_wiring.go
@@ -3,11 +3,13 @@ package cli
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"reflect"
 	"strings"

 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
 	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
+	"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
 	"go.maximumdirect.net/ckconfig"
 	"go.maximumdirect.net/ckprompt"
 	commandkit "go.maximumdirect.net/commandkit"
@@ -16,7 +18,10 @@ import (
 	"go.maximumdirect.net/releasekit/buildinfo"
 )

-type invocationState struct{ engines []engineEntry }
+type invocationState struct {
+	engines []engineEntry
+	client  *http.Client
+}
 type engineEntry struct {
 	selection ckconfig.Selection
 	engine    *pk.Engine
@@ -50,7 +55,11 @@ func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input s
 	if err != nil {
 		return nil, err
 	}
-	runtime, err := llm.NewRuntime(ctx, loaded, nil)
+	var client *http.Client
+	if state != nil {
+		client = state.client
+	}
+	runtime, err := llm.NewRuntime(ctx, loaded, client)
 	if err != nil {
 		return nil, err
 	}
@@ -61,14 +70,37 @@ func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input s
 }

 func buildApplication(extra ...commandkit.Declaration) (*commandkit.App, error) {
+	return buildApplicationWithClient(nil, extra...)
+}
+
+func buildApplicationWithClient(client *http.Client, extra ...commandkit.Declaration) (*commandkit.App, error) {
 	appScope := ckconfig.FromConfigKit("app", "Audita application settings.", loadApplication)
 	appScope.Features.Overrides = true
 	promptScope := ckprompt.ConfigScope(loadPromptEngine)
-	configCommand := ckconfig.Command(ckconfig.Suite[struct{}, *invocationState]{NewState: func() *invocationState { return &invocationState{} }, Scopes: []ckconfig.Scope[struct{}, *invocationState]{appScope, promptScope}})
+	configCommand := ckconfig.Command(ckconfig.Suite[struct{}, *invocationState]{NewState: func() *invocationState { return &invocationState{client: client} }, Scopes: []ckconfig.Scope[struct{}, *invocationState]{appScope, promptScope}})
 	definitions := ckprompt.WithNamespaces(ckprompt.FromEngine(loadPromptEngine), pk.Namespaces{})
-	definitions.NewState = func() *invocationState { return &invocationState{} }
+	definitions.NewState = func() *invocationState { return &invocationState{client: client} }
+	definitions = ckprompt.WithRequestValidation(definitions, loadPromptEngine, func(_ context.Context, seed ckprompt.ValidationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
+		return applicationPromptRequest(seed.Request, seed.Namespaces), nil
+	})
+	definitions = ckprompt.WithRequestPreparation(definitions, loadPromptEngine, func(_ context.Context, seed ckprompt.PreparationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
+		return applicationPromptRequest(seed.Request, seed.Namespaces), nil
+	})
+	definitions = ckprompt.WithSmokeTesting(definitions, loadPromptEngine, nil)
 	declarations := []commandkit.Declaration{configCommand, commandkit.Completion("audita"), processCommand()}
 	declarations = append(declarations, ckprompt.Declarations(definitions)...)
 	declarations = append(declarations, extra...)
 	return commandkit.New(commandkit.Identity{Name: "audita", Version: buildinfo.Version()}, declarations...)
 }
+
+// applicationPromptRequest supplies instructions only for Audita's maintained prompts.
+func applicationPromptRequest(request pk.RunRequest, namespaces *pk.Namespaces) pk.RunRequest {
+	id := request.PromptID
+	if !strings.Contains(id, "/") && namespaces != nil && namespaces.Prompt != "" {
+		id = namespaces.Prompt + "/" + id
+	}
+	if _, ok := prompts.LookupMetadata(id); ok {
+		request.Vars = prompts.WithSharedVariables(request.Vars)
+	}
+	return request
+}
--- a/internal/cli/app_wiring_test.go
+++ b/internal/cli/app_wiring_test.go
@@ -3,12 +3,16 @@ package cli
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"

 	commandkit "go.maximumdirect.net/commandkit"
+	pk "go.maximumdirect.net/promptkit"
 )

 func TestNativeConfigurationAndResourceCommands(t *testing.T) {
@@ -75,3 +79,118 @@ func TestExplicitEmptyConfigurationPathFails(t *testing.T) {
 		t.Fatal("explicit empty path accepted")
 	}
 }
+
+func TestRequestInspectionAndExplicitSmokeUseSameSources(t *testing.T) {
+	dir := t.TempDir()
+	file := filepath.Join(dir, "config.yml")
+	if err := os.WriteFile(file, []byte("version: 3\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	resources := `backends:
+  - id: local
+    endpoint: https://example.invalid/v1
+    smoke_tests:
+      default_profile: fixture
+      authentication:
+        kind: none
+profiles:
+  - id: fixture
+    backend: local
+    model: sample
+collections:
+  audita:
+    prompts:
+      - id: modules.grammar.proposal
+        messages:
+          - role: user
+            content: 'Replacement {{.HardeningPolicy}} {{.SectionJSON}}'
+        output:
+          format: text
+          validation_mode: none
+`
+	if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0644); err != nil {
+		t.Fatal(err)
+	}
+	calls := 0
+	client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		calls++
+		body := `{}`
+		if req.Method == http.MethodPost {
+			requestBody, err := io.ReadAll(req.Body)
+			if err != nil {
+				t.Fatal(err)
+			}
+			content := "PROMPTKIT_OK"
+			if strings.Contains(string(requestBody), "response_format") {
+				content = `{"status":"ok"}`
+			}
+			encoded, err := json.Marshal(content)
+			if err != nil {
+				t.Fatal(err)
+			}
+			body = `{"model":"sample","choices":[{"message":{"content":` + string(encoded) + `}}]}`
+		}
+		return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
+	})}
+	app, err := buildApplicationWithClient(client)
+	if err != nil {
+		t.Fatal(err)
+	}
+	execute := func(args ...string) (commandkit.Outcome, string, string) {
+		t.Helper()
+		var out, stderr bytes.Buffer
+		result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
+		return result, out.String(), stderr.String()
+	}
+	common := []string{"--config", file, "--llm-profile", "fixture"}
+	args := append([]string{"promptkit", "show", "prepared", "audita/modules.grammar.proposal"}, common...)
+	args = append(args, "--llm-var", "SectionJSON=[]", "--llm-var", "HardeningPolicy=custom-instruction")
+	outcome, out, stderr := execute(args...)
+	if outcome.Status != 0 || !strings.Contains(out, "Replacement custom-instruction") {
+		t.Fatalf("preview status=%d out=%q err=%q", outcome.Status, out, stderr)
+	}
+	args = append([]string{"promptkit", "requirements", "validate", "audita/validators.editorial_review"}, common...)
+	args = append(args, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
+	outcome, out, stderr = execute(args...)
+	if outcome.Status != 0 {
+		t.Fatalf("validator status=%d out=%q err=%q", outcome.Status, out, stderr)
+	}
+	validatorPreview := append([]string{"promptkit", "show", "prepared", "audita/validators.editorial_review"}, common...)
+	validatorPreview = append(validatorPreview, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
+	outcome, out, stderr = execute(validatorPreview...)
+	if outcome.Status != 0 || !strings.Contains(out, "teh cat") {
+		t.Fatalf("validator preview status=%d out=%q err=%q", outcome.Status, out, stderr)
+	}
+	if calls != 0 {
+		t.Fatalf("inspection made %d provider calls", calls)
+	}
+	outcome, out, stderr = execute("promptkit", "test", "reachability", "--config", file, "--llm-backend", "local")
+	if outcome.Status != 0 || calls == 0 {
+		t.Fatalf("smoke status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
+	}
+	previous := calls
+	outcome, out, stderr = execute("promptkit", "test", "all", "--config", file, "--llm-backend", "local", "--llm-profile", "fixture", "--format=json")
+	if outcome.Status != 0 || calls <= previous || !strings.Contains(out, "structured_output") || !strings.Contains(out, "text_inference") {
+		t.Fatalf("aggregate status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
+	}
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
+
+func TestRequestDefaultsRespectNamespaceAndCallerVariables(t *testing.T) {
+	request := pk.RunRequest{PromptID: "modules.grammar.proposal", Vars: map[string]string{"HardeningPolicy": "caller"}}
+	result := applicationPromptRequest(request, &pk.Namespaces{Prompt: "audita"})
+	if result.Vars["HardeningPolicy"] != "caller" || result.Vars["ProposalContract"] == "" {
+		t.Fatalf("variables=%v", result.Vars)
+	}
+	if _, changed := request.Vars["ProposalContract"]; changed {
+		t.Fatal("input variables mutated")
+	}
+	generic := pk.RunRequest{PromptID: "generic", Vars: map[string]string{"x": "y"}}
+	result = applicationPromptRequest(generic, nil)
+	if len(result.Vars) != 1 || result.Vars["x"] != "y" {
+		t.Fatalf("generic request changed: %v", result.Vars)
+	}
+}