Expose native PromptKit discovery preparation and smoke testing

This commit is contained in:
2026-11-20 04:37:18 +00:00
parent f2f807e87b
commit f9b5121432
4 changed files with 421 additions and 1 deletions

View File

@@ -380,6 +380,25 @@ Verify app overrides are available and framework overrides are not emulated.
## Stage 4: Install Native PromptKit Resources, Requirements, And Preview
**Status: Complete.** Composed all twelve native callbacks, empty-default
namespace views, and native convenience declarations. Request builders remain
nil so native seeds pass through unchanged. Every callback carries application
acquisition evidence separately, preserving framework reports and typed errors.
Verified on Go 1.27.4 with focused real-library tests, race checks, and vet:
exact required and undeclared input names, optional/conditional dependencies,
explicit-empty variable presence, complete versus partial reports and structured
issue codes, qualified prompt/profile identities, namespace selection, selection
failures, and discovery without artifacts, schemas, or generation credentials.
Also checked five inventories and native default/filter behavior, three named
inspections, content-bearing prepared reports in automatic/text/YAML/JSON modes,
and concrete validation/preparation without generation. A loopback-only provider
verified explicit smoke execution and native unsuccessful findings with status
zero, without disclosing provider payloads. Narratio's shared-context discovery,
validation, and preview coverage is present; execution consistency is checked
with positional generation in Stage 5 and the assembled workflow in Stage 9.
Legacy-file compilation cleanup remains deferred to Stage 8.
**Primary files:** CLI PromptKit-suite composition, engine-loading adapters,
and behavior tests replacing custom definition/prepared inspection tests.

View File

@@ -20,7 +20,7 @@ func Execute(ctx context.Context, identity commandkit.Identity, args []string, s
}
func newApp(identity commandkit.Identity) (*commandkit.App, error) {
definitions := ckprompt.FromEngine(loadEngine)
definitions := promptSuite()
declarations := []commandkit.Declaration{
commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Summary: "Run a stored prompt.", Run: commandNotImplemented}),
commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Summary: "Serve the HTTP API.", Flags: serveFlags(), Run: func(commandkit.Invocation, serveInput) commandkit.Outcome {

View File

@@ -0,0 +1,40 @@
package cli
import (
"context"
"go.maximumdirect.net/ckprompt"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func promptSuite() ckprompt.Suite[applicationInput, *acquisitionState] {
suite := ckprompt.FromEngine(loadEngine)
suite.NewInput = func() applicationInput { return applicationInput{} }
suite.NewState = func() *acquisitionState { return &acquisitionState{} }
suite = ckprompt.WithRequestValidation(suite, loadEngine, nil)
suite = ckprompt.WithRequestPreparation(suite, loadEngine, nil)
suite = ckprompt.WithSmokeTesting(suite, loadEngine, nil)
suite = ckprompt.WithNamespaces(suite, promptkit.Namespaces{})
suite.ListPrompts = withApplicationEvidence(suite.ListPrompts)
suite.ListProfiles = withApplicationEvidence(suite.ListProfiles)
suite.ListBackends = withApplicationEvidence(suite.ListBackends)
suite.ListNamespaces = withApplicationEvidence(suite.ListNamespaces)
suite.ListCollections = withApplicationEvidence(suite.ListCollections)
suite.InspectPrompt = withApplicationEvidence(suite.InspectPrompt)
suite.InspectProfile = withApplicationEvidence(suite.InspectProfile)
suite.InspectBackend = withApplicationEvidence(suite.InspectBackend)
suite.InspectRequirements = withApplicationEvidence(suite.InspectRequirements)
suite.ValidateRequest = withApplicationEvidence(suite.ValidateRequest)
suite.PrepareRequest = withApplicationEvidence(suite.PrepareRequest)
suite.SmokeTest = withApplicationEvidence(suite.SmokeTest)
return suite
}
func withApplicationEvidence[R, V any](callback func(context.Context, R, applicationInput, *acquisitionState) (commandkit.Acquired[V], error)) func(context.Context, R, applicationInput, *acquisitionState) (commandkit.Acquired[V], error) {
return func(ctx context.Context, request R, input applicationInput, state *acquisitionState) (commandkit.Acquired[V], error) {
start := state.mark()
acquired, err := callback(ctx, request, input, state)
return attachApplicationEvidence(acquired, err, state, start, false)
}
}

View File

@@ -0,0 +1,361 @@
package cli
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync/atomic"
"testing"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func nativePromptFixture(t *testing.T, endpoint string) {
t.Helper()
t.Chdir(t.TempDir())
writeNativeFile(t, "app.yml", "promptkit:\n configuration: framework.yml\n schema_dir: nonexistent-schemas\ntransient_retries: 0\n")
writeNativeFile(t, "framework.yml", fmt.Sprintf(`collections:
support:
release_version: '1'
backends:
- id: local
endpoint: %q
profiles:
- id: writer
backend: support/local
model: controlled-model
- id: protected
backend: support/local
model: controlled-model
api_key_required: true
prompts:
- id: summary
default_profile: support/protected
session_id: '{{.session_variable}}'
inputs:
- name: transcript
required: true
content_type: text/plain
description: Exact transcript binding
- name: glossary
required: false
description: ''
messages:
- role: user
content: 'template-canary {{input "transcript"}} {{input "unlisted"}} {{if .mode}}{{input "glossary"}}{{end}} {{.title}}'
output:
format: text
validation_mode: none
- id: dynamic
default_profile: support/protected
messages:
- role: user
content: '{{input .source}}'
output:
format: text
validation_mode: none
- id: unsupported
default_profile: support/protected
messages:
- role: user
content: '{{define "child"}}{{input "called"}}{{end}}{{template "child" .}}'
output:
format: text
validation_mode: none
- id: schema-needed
default_profile: support/protected
messages:
- role: user
content: 'Return JSON'
output:
format: json
validation_mode: json_schema
schema_path: missing.json
review:
release_version: '1'
profiles:
- id: writer
backend: support/local
model: alternate-model
prompts:
- id: summary
default_profile: review/writer
inputs:
- name: memo
required: true
messages:
- role: user
content: '{{input "memo"}}'
output:
format: text
validation_mode: none
`, endpoint))
}
type consumerRequirements struct {
PromptID string `json:"prompt_id"`
ProfileID string `json:"profile_id"`
Inputs []struct {
Name string `json:"name"`
Kind promptkit.RequirementKind `json:"kind"`
Declared bool `json:"declared"`
ContentType *string `json:"content_type"`
Description *string `json:"description"`
} `json:"inputs"`
Variables []struct {
Name string `json:"name"`
Supplied bool `json:"supplied"`
} `json:"variables"`
Complete bool `json:"complete"`
Issues []struct {
Code promptkit.RequirementIssueCode `json:"code"`
} `json:"issues"`
}
func discoverNative(t *testing.T, prompt string, flags ...string) consumerRequirements {
t.Helper()
args := []string{"promptkit", "requirements", "show", prompt, "--config=app.yml", "--format=json"}
args = append(args, flags...)
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 {
t.Fatalf("discovery: %+v %q %q", outcome, output, diagnostics)
}
var report consumerRequirements
if err := json.Unmarshal([]byte(output), &report); err != nil {
t.Fatal(err)
}
for _, excluded := range []string{"template-canary", "unrelated-variable-value", "schema_version"} {
if strings.Contains(output, excluded) {
t.Fatalf("unexpected discovery content %q: %s", excluded, output)
}
}
return report
}
func TestRequirementsExposeExactBindingsBeforeArtifactsExist(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
report := discoverNative(t, "support/summary", "--var=mode=", "--var=unrelated=unrelated-variable-value", "--session-id=session")
if !report.Complete || report.PromptID != "support/summary" || report.ProfileID != "support/protected" {
t.Fatalf("discovery identity/completeness: %+v", report)
}
var names []string
for _, input := range report.Inputs {
names = append(names, input.Name)
switch input.Name {
case "transcript":
if input.Kind != promptkit.RequirementRequired || !input.Declared || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil {
t.Fatalf("required metadata: %+v", input)
}
case "unlisted":
if input.Kind != promptkit.RequirementRequired || input.Declared {
t.Fatalf("undeclared reference: %+v", input)
}
case "glossary":
if input.Kind != promptkit.RequirementOptional || input.Description == nil || *input.Description != "" {
t.Fatalf("optional metadata: %+v", input)
}
}
}
if !reflect.DeepEqual(names, []string{"glossary", "transcript", "unlisted"}) {
t.Fatalf("exact input names: %v", names)
}
var emptySupplied, absent bool
for _, variable := range report.Variables {
if variable.Name == "mode" {
emptySupplied = variable.Supplied
}
if variable.Name == "title" {
absent = !variable.Supplied
}
}
if !emptySupplied || !absent {
t.Fatalf("variable presence: %+v", report.Variables)
}
schemaIndependent := discoverNative(t, "support/schema-needed")
if !schemaIndependent.Complete || len(schemaIndependent.Inputs) != 0 {
t.Fatalf("schema-free discovery: %+v", schemaIndependent)
}
// Complete discovery does not assert that these required bindings exist.
outcome, _, diagnostics := executeNative(t, "promptkit", "requirements", "validate", "support/summary", "--config=app.yml", "--profile=support/writer", "--var=mode=", "--var=title=Title", "--session-id=session")
if outcome.Status == 0 || diagnostics == "" {
t.Fatalf("unbound request validated: %+v %q", outcome, diagnostics)
}
outcome, _, diagnostics = executeNative(t, "promptkit", "show", "prepared", "support/schema-needed", "--config=app.yml", "--profile=support/writer")
if outcome.Status == 0 || diagnostics == "" {
t.Fatalf("missing preparation schema accepted: %+v %q", outcome, diagnostics)
}
}
func TestRequirementsPreserveConditionalAndPartialDiscovery(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
unknown := discoverNative(t, "support/summary", "--session-id=session")
if unknown.Complete || len(unknown.Issues) == 0 {
t.Fatalf("unknown condition claimed complete: %+v", unknown)
}
for _, input := range unknown.Inputs {
if input.Name == "glossary" && input.Kind != promptkit.RequirementConditional {
t.Fatalf("unknown branch: %+v", input)
}
}
known := discoverNative(t, "support/summary", "--var=mode=include", "--session-id=session")
if !known.Complete {
t.Fatalf("known branch incomplete: %+v", known)
}
for _, input := range known.Inputs {
if input.Name == "glossary" && input.Kind != promptkit.RequirementRequired {
t.Fatalf("known branch: %+v", input)
}
}
dynamic := discoverNative(t, "support/dynamic")
if dynamic.Complete || len(dynamic.Issues) == 0 {
t.Fatalf("dynamic name not partial: %+v", dynamic)
}
resolved := discoverNative(t, "support/dynamic", "--var=source=exact,name")
if !resolved.Complete || len(resolved.Inputs) != 1 || resolved.Inputs[0].Name != "exact,name" || resolved.Inputs[0].Declared {
t.Fatalf("resolved identifier: %+v", resolved)
}
unsupported := discoverNative(t, "support/unsupported")
var found bool
for _, issue := range unsupported.Issues {
found = found || issue.Code == promptkit.RequirementIssueUnsupportedConstruct
}
if unsupported.Complete || !found {
t.Fatalf("unsupported analysis not preserved: %+v", unsupported)
}
}
func TestRequirementsSelectionUsesNativeNamespacesAndFailures(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
before := discoverNative(t, "summary", "--prompt-namespace=support", "--var=mode=", "--session-id=session")
after := discoverNative(t, "support/summary", "--prompt-namespace=review", "--profile=writer", "--profile-namespace=review", "--var=mode=", "--session-id=session")
if after.PromptID != "support/summary" || after.ProfileID != "review/writer" || !reflect.DeepEqual(before.Inputs, after.Inputs) {
t.Fatalf("identity/definition changed: before=%+v after=%+v", before, after)
}
other := discoverNative(t, "review/summary")
if other.PromptID != "review/summary" || other.ProfileID != "review/writer" || len(other.Inputs) != 1 || other.Inputs[0].Name != "memo" {
t.Fatalf("other definition: %+v", other)
}
for _, args := range [][]string{
{"promptkit", "requirements", "show", "missing", "--config=app.yml"},
{"promptkit", "requirements", "show", "support/summary", "--config=app.yml", "--profile=missing"},
} {
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status == 0 || output != "" || diagnostics == "" {
t.Fatalf("selection failure: %+v %q %q", outcome, output, diagnostics)
}
}
}
func TestNativeInventoriesInspectionAndPreparationDoNotGenerate(t *testing.T) {
var calls atomic.Int64
var fail atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.Header().Set("Content-Type", "application/json")
if fail.Load() {
w.WriteHeader(http.StatusServiceUnavailable)
fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`)
return
}
fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"controlled"}}]}`)
}))
defer server.Close()
nativePromptFixture(t, server.URL+"/v1")
for _, args := range [][]string{
{"promptkit", "list", "prompts", "--all-namespaces"},
{"promptkit", "list", "profiles", "--namespace=support"},
{"promptkit", "list", "backends", "--namespace=support"},
{"promptkit", "list", "namespaces"},
{"promptkit", "list", "collections"},
{"promptkit", "show", "prompt", "support/summary"},
{"promptkit", "show", "profile", "support/writer"},
{"promptkit", "show", "backend", "support/local"},
} {
args = append(args, "--config=app.yml", "--format=json")
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 || output == "" {
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
}
var value any
if err := json.Unmarshal([]byte(output), &value); err != nil {
t.Fatal(err)
}
if args[1] == "show" && args[2] == "prompt" && !strings.Contains(output, "template-canary") {
t.Fatalf("native prompt content omitted: %s", output)
}
}
engine, err := loadEngine(context.Background(), ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "app.yml", Set: true}}, applicationInput{}, nil)
if err != nil {
t.Fatal(err)
}
nativePrompts, err := engine.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
outcome, output, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--config=app.yml", "--format=json")
var listed []promptkit.PromptIdentity
if err := json.Unmarshal([]byte(output), &listed); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || !reflect.DeepEqual(listed, nativePrompts) {
t.Fatalf("native default listing changed: %+v %q %q", outcome, output, diagnostics)
}
outcome, output, diagnostics = executeNative(t, "promptkit", "list", "prompts", "--namespace=review", "--config=app.yml", "--format=json")
if outcome.Status != 0 || strings.Contains(output, "support/summary") || !strings.Contains(output, "review/summary") {
t.Fatalf("namespace filter lost: %+v %q %q", outcome, output, diagnostics)
}
writeNativeFile(t, "transcript.md", "artifact-payload-canary")
flags := []string{"summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--session-id=session", "--var=mode=", "--var=title=Title", "--input-file=transcript=transcript.md", "--input=unlisted=literal"}
for _, command := range [][]string{{"promptkit", "requirements", "validate"}, {"promptkit", "show", "prepared"}} {
args := append(append([]string{}, command...), flags...)
outcome, output, diagnostics := executeNative(t, append(args, "--format=json")...)
if outcome.Status != 0 {
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
}
if command[1] == "show" && (!strings.Contains(output, "artifact-payload-canary") || !strings.Contains(output, "support/writer") || !strings.Contains(output, "support/summary")) {
t.Fatalf("native prepared content/context lost: %s", output)
}
}
for _, format := range []string{"", "text", "yaml"} {
args := append([]string{"promptkit", "show", "prepared"}, flags...)
if format != "" {
args = append(args, "--format="+format)
}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 || !strings.Contains(output, "artifact-payload-canary") {
t.Fatalf("native preview %q: %+v %q %q", format, outcome, output, diagnostics)
}
}
if calls.Load() != 0 {
t.Fatal("inspection/preparation contacted provider")
}
// Smoke tests are the explicit provider-effect boundary. Native findings,
// not an application readiness policy, determine what the report says.
outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=writer", "--profile-namespace=support", "--format=json")
if outcome.Status != 0 || calls.Load() != 1 || !strings.Contains(output, "text_inference") {
t.Fatalf("controlled smoke: %+v %q %q calls=%d", outcome, output, diagnostics, calls.Load())
}
fail.Store(true)
outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=support/writer", "--format=json")
var findings struct {
Completed bool `json:"completed"`
Failure string `json:"failure"`
}
if err := json.Unmarshal([]byte(output), &findings); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || !findings.Completed || findings.Failure == "" || strings.Contains(output+diagnostics, "provider-private-canary") {
t.Fatalf("native unsuccessful finding changed: %+v %q %q", outcome, output, diagnostics)
}
writeNativeFile(t, "invalid-app.yml", "server:\n max_request_bytes: -1\n")
outcome, output, diagnostics = executeNative(t, "promptkit", "requirements", "show", "support/summary", "--config=invalid-app.yml", "--quiet")
if outcome.Kind != commandkit.OutcomeUsage || output != "" || !strings.Contains(diagnostics, "Application configuration") {
t.Fatalf("acquisition evidence: %+v %q %q", outcome, output, diagnostics)
}
}