Files
scriptorium/internal/adapter/cli/native_prompt_test.go

362 lines
14 KiB
Go
Raw Normal View History

package cli
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync/atomic"
"testing"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func nativePromptFixture(t *testing.T, endpoint string) {
t.Helper()
t.Chdir(t.TempDir())
writeNativeFile(t, "app.yml", "promptkit:\n configuration: framework.yml\n schema_dir: nonexistent-schemas\ntransient_retries: 0\n")
writeNativeFile(t, "framework.yml", fmt.Sprintf(`collections:
support:
release_version: '1'
backends:
- id: local
endpoint: %q
profiles:
- id: writer
backend: support/local
model: controlled-model
- id: protected
backend: support/local
model: controlled-model
api_key_required: true
prompts:
- id: summary
default_profile: support/protected
session_id: '{{.session_variable}}'
inputs:
- name: transcript
required: true
content_type: text/plain
description: Exact transcript binding
- name: glossary
required: false
description: ''
messages:
- role: user
content: 'template-canary {{input "transcript"}} {{input "unlisted"}} {{if .mode}}{{input "glossary"}}{{end}} {{.title}}'
output:
format: text
validation_mode: none
- id: dynamic
default_profile: support/protected
messages:
- role: user
content: '{{input .source}}'
output:
format: text
validation_mode: none
- id: unsupported
default_profile: support/protected
messages:
- role: user
content: '{{define "child"}}{{input "called"}}{{end}}{{template "child" .}}'
output:
format: text
validation_mode: none
- id: schema-needed
default_profile: support/protected
messages:
- role: user
content: 'Return JSON'
output:
format: json
validation_mode: json_schema
schema_path: missing.json
review:
release_version: '1'
profiles:
- id: writer
backend: support/local
model: alternate-model
prompts:
- id: summary
default_profile: review/writer
inputs:
- name: memo
required: true
messages:
- role: user
content: '{{input "memo"}}'
output:
format: text
validation_mode: none
`, endpoint))
}
type consumerRequirements struct {
PromptID string `json:"prompt_id"`
ProfileID string `json:"profile_id"`
Inputs []struct {
Name string `json:"name"`
Kind promptkit.RequirementKind `json:"kind"`
Declared bool `json:"declared"`
ContentType *string `json:"content_type"`
Description *string `json:"description"`
} `json:"inputs"`
Variables []struct {
Name string `json:"name"`
Supplied bool `json:"supplied"`
} `json:"variables"`
Complete bool `json:"complete"`
Issues []struct {
Code promptkit.RequirementIssueCode `json:"code"`
} `json:"issues"`
}
func discoverNative(t *testing.T, prompt string, flags ...string) consumerRequirements {
t.Helper()
args := []string{"promptkit", "requirements", "show", prompt, "--config=app.yml", "--format=json"}
args = append(args, flags...)
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 {
t.Fatalf("discovery: %+v %q %q", outcome, output, diagnostics)
}
var report consumerRequirements
if err := json.Unmarshal([]byte(output), &report); err != nil {
t.Fatal(err)
}
for _, excluded := range []string{"template-canary", "unrelated-variable-value", "schema_version"} {
if strings.Contains(output, excluded) {
t.Fatalf("unexpected discovery content %q: %s", excluded, output)
}
}
return report
}
func TestRequirementsExposeExactBindingsBeforeArtifactsExist(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
report := discoverNative(t, "support/summary", "--var=mode=", "--var=unrelated=unrelated-variable-value", "--session-id=session")
if !report.Complete || report.PromptID != "support/summary" || report.ProfileID != "support/protected" {
t.Fatalf("discovery identity/completeness: %+v", report)
}
var names []string
for _, input := range report.Inputs {
names = append(names, input.Name)
switch input.Name {
case "transcript":
if input.Kind != promptkit.RequirementRequired || !input.Declared || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil {
t.Fatalf("required metadata: %+v", input)
}
case "unlisted":
if input.Kind != promptkit.RequirementRequired || input.Declared {
t.Fatalf("undeclared reference: %+v", input)
}
case "glossary":
if input.Kind != promptkit.RequirementOptional || input.Description == nil || *input.Description != "" {
t.Fatalf("optional metadata: %+v", input)
}
}
}
if !reflect.DeepEqual(names, []string{"glossary", "transcript", "unlisted"}) {
t.Fatalf("exact input names: %v", names)
}
var emptySupplied, absent bool
for _, variable := range report.Variables {
if variable.Name == "mode" {
emptySupplied = variable.Supplied
}
if variable.Name == "title" {
absent = !variable.Supplied
}
}
if !emptySupplied || !absent {
t.Fatalf("variable presence: %+v", report.Variables)
}
schemaIndependent := discoverNative(t, "support/schema-needed")
if !schemaIndependent.Complete || len(schemaIndependent.Inputs) != 0 {
t.Fatalf("schema-free discovery: %+v", schemaIndependent)
}
// Complete discovery does not assert that these required bindings exist.
outcome, _, diagnostics := executeNative(t, "promptkit", "requirements", "validate", "support/summary", "--config=app.yml", "--profile=support/writer", "--var=mode=", "--var=title=Title", "--session-id=session")
if outcome.Status == 0 || diagnostics == "" {
t.Fatalf("unbound request validated: %+v %q", outcome, diagnostics)
}
outcome, _, diagnostics = executeNative(t, "promptkit", "show", "prepared", "support/schema-needed", "--config=app.yml", "--profile=support/writer")
if outcome.Status == 0 || diagnostics == "" {
t.Fatalf("missing preparation schema accepted: %+v %q", outcome, diagnostics)
}
}
func TestRequirementsPreserveConditionalAndPartialDiscovery(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
unknown := discoverNative(t, "support/summary", "--session-id=session")
if unknown.Complete || len(unknown.Issues) == 0 {
t.Fatalf("unknown condition claimed complete: %+v", unknown)
}
for _, input := range unknown.Inputs {
if input.Name == "glossary" && input.Kind != promptkit.RequirementConditional {
t.Fatalf("unknown branch: %+v", input)
}
}
known := discoverNative(t, "support/summary", "--var=mode=include", "--session-id=session")
if !known.Complete {
t.Fatalf("known branch incomplete: %+v", known)
}
for _, input := range known.Inputs {
if input.Name == "glossary" && input.Kind != promptkit.RequirementRequired {
t.Fatalf("known branch: %+v", input)
}
}
dynamic := discoverNative(t, "support/dynamic")
if dynamic.Complete || len(dynamic.Issues) == 0 {
t.Fatalf("dynamic name not partial: %+v", dynamic)
}
resolved := discoverNative(t, "support/dynamic", "--var=source=exact,name")
if !resolved.Complete || len(resolved.Inputs) != 1 || resolved.Inputs[0].Name != "exact,name" || resolved.Inputs[0].Declared {
t.Fatalf("resolved identifier: %+v", resolved)
}
unsupported := discoverNative(t, "support/unsupported")
var found bool
for _, issue := range unsupported.Issues {
found = found || issue.Code == promptkit.RequirementIssueUnsupportedConstruct
}
if unsupported.Complete || !found {
t.Fatalf("unsupported analysis not preserved: %+v", unsupported)
}
}
func TestRequirementsSelectionUsesNativeNamespacesAndFailures(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
before := discoverNative(t, "summary", "--prompt-namespace=support", "--var=mode=", "--session-id=session")
after := discoverNative(t, "support/summary", "--prompt-namespace=review", "--profile=writer", "--profile-namespace=review", "--var=mode=", "--session-id=session")
if after.PromptID != "support/summary" || after.ProfileID != "review/writer" || !reflect.DeepEqual(before.Inputs, after.Inputs) {
t.Fatalf("identity/definition changed: before=%+v after=%+v", before, after)
}
other := discoverNative(t, "review/summary")
if other.PromptID != "review/summary" || other.ProfileID != "review/writer" || len(other.Inputs) != 1 || other.Inputs[0].Name != "memo" {
t.Fatalf("other definition: %+v", other)
}
for _, args := range [][]string{
{"promptkit", "requirements", "show", "missing", "--config=app.yml"},
{"promptkit", "requirements", "show", "support/summary", "--config=app.yml", "--profile=missing"},
} {
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status == 0 || output != "" || diagnostics == "" {
t.Fatalf("selection failure: %+v %q %q", outcome, output, diagnostics)
}
}
}
func TestNativeInventoriesInspectionAndPreparationDoNotGenerate(t *testing.T) {
var calls atomic.Int64
var fail atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.Header().Set("Content-Type", "application/json")
if fail.Load() {
w.WriteHeader(http.StatusServiceUnavailable)
fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`)
return
}
fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"controlled"}}]}`)
}))
defer server.Close()
nativePromptFixture(t, server.URL+"/v1")
for _, args := range [][]string{
{"promptkit", "list", "prompts", "--all-namespaces"},
{"promptkit", "list", "profiles", "--namespace=support"},
{"promptkit", "list", "backends", "--namespace=support"},
{"promptkit", "list", "namespaces"},
{"promptkit", "list", "collections"},
{"promptkit", "show", "prompt", "support/summary"},
{"promptkit", "show", "profile", "support/writer"},
{"promptkit", "show", "backend", "support/local"},
} {
args = append(args, "--config=app.yml", "--format=json")
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 || output == "" {
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
}
var value any
if err := json.Unmarshal([]byte(output), &value); err != nil {
t.Fatal(err)
}
if args[1] == "show" && args[2] == "prompt" && !strings.Contains(output, "template-canary") {
t.Fatalf("native prompt content omitted: %s", output)
}
}
engine, err := loadEngine(context.Background(), ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "app.yml", Set: true}}, applicationInput{}, nil)
if err != nil {
t.Fatal(err)
}
nativePrompts, err := engine.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
outcome, output, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--config=app.yml", "--format=json")
var listed []promptkit.PromptIdentity
if err := json.Unmarshal([]byte(output), &listed); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || !reflect.DeepEqual(listed, nativePrompts) {
t.Fatalf("native default listing changed: %+v %q %q", outcome, output, diagnostics)
}
outcome, output, diagnostics = executeNative(t, "promptkit", "list", "prompts", "--namespace=review", "--config=app.yml", "--format=json")
if outcome.Status != 0 || strings.Contains(output, "support/summary") || !strings.Contains(output, "review/summary") {
t.Fatalf("namespace filter lost: %+v %q %q", outcome, output, diagnostics)
}
writeNativeFile(t, "transcript.md", "artifact-payload-canary")
flags := []string{"summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--session-id=session", "--var=mode=", "--var=title=Title", "--input-file=transcript=transcript.md", "--input=unlisted=literal"}
for _, command := range [][]string{{"promptkit", "requirements", "validate"}, {"promptkit", "show", "prepared"}} {
args := append(append([]string{}, command...), flags...)
outcome, output, diagnostics := executeNative(t, append(args, "--format=json")...)
if outcome.Status != 0 {
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
}
if command[1] == "show" && (!strings.Contains(output, "artifact-payload-canary") || !strings.Contains(output, "support/writer") || !strings.Contains(output, "support/summary")) {
t.Fatalf("native prepared content/context lost: %s", output)
}
}
for _, format := range []string{"", "text", "yaml"} {
args := append([]string{"promptkit", "show", "prepared"}, flags...)
if format != "" {
args = append(args, "--format="+format)
}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 || !strings.Contains(output, "artifact-payload-canary") {
t.Fatalf("native preview %q: %+v %q %q", format, outcome, output, diagnostics)
}
}
if calls.Load() != 0 {
t.Fatal("inspection/preparation contacted provider")
}
// Smoke tests are the explicit provider-effect boundary. Native findings,
// not an application readiness policy, determine what the report says.
outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=writer", "--profile-namespace=support", "--format=json")
if outcome.Status != 0 || calls.Load() != 1 || !strings.Contains(output, "text_inference") {
t.Fatalf("controlled smoke: %+v %q %q calls=%d", outcome, output, diagnostics, calls.Load())
}
fail.Store(true)
outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=support/writer", "--format=json")
var findings struct {
Completed bool `json:"completed"`
Failure string `json:"failure"`
}
if err := json.Unmarshal([]byte(output), &findings); err != nil {
t.Fatal(err)
}
if outcome.Status != 0 || !findings.Completed || findings.Failure == "" || strings.Contains(output+diagnostics, "provider-private-canary") {
t.Fatalf("native unsuccessful finding changed: %+v %q %q", outcome, output, diagnostics)
}
writeNativeFile(t, "invalid-app.yml", "server:\n max_request_bytes: -1\n")
outcome, output, diagnostics = executeNative(t, "promptkit", "requirements", "show", "support/summary", "--config=invalid-app.yml", "--quiet")
if outcome.Kind != commandkit.OutcomeUsage || output != "" || !strings.Contains(diagnostics, "Application configuration") {
t.Fatalf("acquisition evidence: %+v %q %q", outcome, output, diagnostics)
}
}