362 lines
14 KiB
Go
362 lines
14 KiB
Go
|
|
package cli
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"net/http"
|
||
|
|
"net/http/httptest"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"sync/atomic"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"go.maximumdirect.net/ckconfig"
|
||
|
|
"go.maximumdirect.net/commandkit"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
func nativePromptFixture(t *testing.T, endpoint string) {
|
||
|
|
t.Helper()
|
||
|
|
t.Chdir(t.TempDir())
|
||
|
|
writeNativeFile(t, "app.yml", "promptkit:\n configuration: framework.yml\n schema_dir: nonexistent-schemas\ntransient_retries: 0\n")
|
||
|
|
writeNativeFile(t, "framework.yml", fmt.Sprintf(`collections:
|
||
|
|
support:
|
||
|
|
release_version: '1'
|
||
|
|
backends:
|
||
|
|
- id: local
|
||
|
|
endpoint: %q
|
||
|
|
profiles:
|
||
|
|
- id: writer
|
||
|
|
backend: support/local
|
||
|
|
model: controlled-model
|
||
|
|
- id: protected
|
||
|
|
backend: support/local
|
||
|
|
model: controlled-model
|
||
|
|
api_key_required: true
|
||
|
|
prompts:
|
||
|
|
- id: summary
|
||
|
|
default_profile: support/protected
|
||
|
|
session_id: '{{.session_variable}}'
|
||
|
|
inputs:
|
||
|
|
- name: transcript
|
||
|
|
required: true
|
||
|
|
content_type: text/plain
|
||
|
|
description: Exact transcript binding
|
||
|
|
- name: glossary
|
||
|
|
required: false
|
||
|
|
description: ''
|
||
|
|
messages:
|
||
|
|
- role: user
|
||
|
|
content: 'template-canary {{input "transcript"}} {{input "unlisted"}} {{if .mode}}{{input "glossary"}}{{end}} {{.title}}'
|
||
|
|
output:
|
||
|
|
format: text
|
||
|
|
validation_mode: none
|
||
|
|
- id: dynamic
|
||
|
|
default_profile: support/protected
|
||
|
|
messages:
|
||
|
|
- role: user
|
||
|
|
content: '{{input .source}}'
|
||
|
|
output:
|
||
|
|
format: text
|
||
|
|
validation_mode: none
|
||
|
|
- id: unsupported
|
||
|
|
default_profile: support/protected
|
||
|
|
messages:
|
||
|
|
- role: user
|
||
|
|
content: '{{define "child"}}{{input "called"}}{{end}}{{template "child" .}}'
|
||
|
|
output:
|
||
|
|
format: text
|
||
|
|
validation_mode: none
|
||
|
|
- id: schema-needed
|
||
|
|
default_profile: support/protected
|
||
|
|
messages:
|
||
|
|
- role: user
|
||
|
|
content: 'Return JSON'
|
||
|
|
output:
|
||
|
|
format: json
|
||
|
|
validation_mode: json_schema
|
||
|
|
schema_path: missing.json
|
||
|
|
review:
|
||
|
|
release_version: '1'
|
||
|
|
profiles:
|
||
|
|
- id: writer
|
||
|
|
backend: support/local
|
||
|
|
model: alternate-model
|
||
|
|
prompts:
|
||
|
|
- id: summary
|
||
|
|
default_profile: review/writer
|
||
|
|
inputs:
|
||
|
|
- name: memo
|
||
|
|
required: true
|
||
|
|
messages:
|
||
|
|
- role: user
|
||
|
|
content: '{{input "memo"}}'
|
||
|
|
output:
|
||
|
|
format: text
|
||
|
|
validation_mode: none
|
||
|
|
`, endpoint))
|
||
|
|
}
|
||
|
|
|
||
|
|
type consumerRequirements struct {
|
||
|
|
PromptID string `json:"prompt_id"`
|
||
|
|
ProfileID string `json:"profile_id"`
|
||
|
|
Inputs []struct {
|
||
|
|
Name string `json:"name"`
|
||
|
|
Kind promptkit.RequirementKind `json:"kind"`
|
||
|
|
Declared bool `json:"declared"`
|
||
|
|
ContentType *string `json:"content_type"`
|
||
|
|
Description *string `json:"description"`
|
||
|
|
} `json:"inputs"`
|
||
|
|
Variables []struct {
|
||
|
|
Name string `json:"name"`
|
||
|
|
Supplied bool `json:"supplied"`
|
||
|
|
} `json:"variables"`
|
||
|
|
Complete bool `json:"complete"`
|
||
|
|
Issues []struct {
|
||
|
|
Code promptkit.RequirementIssueCode `json:"code"`
|
||
|
|
} `json:"issues"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func discoverNative(t *testing.T, prompt string, flags ...string) consumerRequirements {
|
||
|
|
t.Helper()
|
||
|
|
args := []string{"promptkit", "requirements", "show", prompt, "--config=app.yml", "--format=json"}
|
||
|
|
args = append(args, flags...)
|
||
|
|
outcome, output, diagnostics := executeNative(t, args...)
|
||
|
|
if outcome.Status != 0 {
|
||
|
|
t.Fatalf("discovery: %+v %q %q", outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
var report consumerRequirements
|
||
|
|
if err := json.Unmarshal([]byte(output), &report); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
for _, excluded := range []string{"template-canary", "unrelated-variable-value", "schema_version"} {
|
||
|
|
if strings.Contains(output, excluded) {
|
||
|
|
t.Fatalf("unexpected discovery content %q: %s", excluded, output)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return report
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRequirementsExposeExactBindingsBeforeArtifactsExist(t *testing.T) {
|
||
|
|
nativePromptFixture(t, "http://127.0.0.1:1/v1")
|
||
|
|
report := discoverNative(t, "support/summary", "--var=mode=", "--var=unrelated=unrelated-variable-value", "--session-id=session")
|
||
|
|
if !report.Complete || report.PromptID != "support/summary" || report.ProfileID != "support/protected" {
|
||
|
|
t.Fatalf("discovery identity/completeness: %+v", report)
|
||
|
|
}
|
||
|
|
var names []string
|
||
|
|
for _, input := range report.Inputs {
|
||
|
|
names = append(names, input.Name)
|
||
|
|
switch input.Name {
|
||
|
|
case "transcript":
|
||
|
|
if input.Kind != promptkit.RequirementRequired || !input.Declared || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil {
|
||
|
|
t.Fatalf("required metadata: %+v", input)
|
||
|
|
}
|
||
|
|
case "unlisted":
|
||
|
|
if input.Kind != promptkit.RequirementRequired || input.Declared {
|
||
|
|
t.Fatalf("undeclared reference: %+v", input)
|
||
|
|
}
|
||
|
|
case "glossary":
|
||
|
|
if input.Kind != promptkit.RequirementOptional || input.Description == nil || *input.Description != "" {
|
||
|
|
t.Fatalf("optional metadata: %+v", input)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if !reflect.DeepEqual(names, []string{"glossary", "transcript", "unlisted"}) {
|
||
|
|
t.Fatalf("exact input names: %v", names)
|
||
|
|
}
|
||
|
|
var emptySupplied, absent bool
|
||
|
|
for _, variable := range report.Variables {
|
||
|
|
if variable.Name == "mode" {
|
||
|
|
emptySupplied = variable.Supplied
|
||
|
|
}
|
||
|
|
if variable.Name == "title" {
|
||
|
|
absent = !variable.Supplied
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if !emptySupplied || !absent {
|
||
|
|
t.Fatalf("variable presence: %+v", report.Variables)
|
||
|
|
}
|
||
|
|
schemaIndependent := discoverNative(t, "support/schema-needed")
|
||
|
|
if !schemaIndependent.Complete || len(schemaIndependent.Inputs) != 0 {
|
||
|
|
t.Fatalf("schema-free discovery: %+v", schemaIndependent)
|
||
|
|
}
|
||
|
|
// Complete discovery does not assert that these required bindings exist.
|
||
|
|
outcome, _, diagnostics := executeNative(t, "promptkit", "requirements", "validate", "support/summary", "--config=app.yml", "--profile=support/writer", "--var=mode=", "--var=title=Title", "--session-id=session")
|
||
|
|
if outcome.Status == 0 || diagnostics == "" {
|
||
|
|
t.Fatalf("unbound request validated: %+v %q", outcome, diagnostics)
|
||
|
|
}
|
||
|
|
outcome, _, diagnostics = executeNative(t, "promptkit", "show", "prepared", "support/schema-needed", "--config=app.yml", "--profile=support/writer")
|
||
|
|
if outcome.Status == 0 || diagnostics == "" {
|
||
|
|
t.Fatalf("missing preparation schema accepted: %+v %q", outcome, diagnostics)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRequirementsPreserveConditionalAndPartialDiscovery(t *testing.T) {
|
||
|
|
nativePromptFixture(t, "http://127.0.0.1:1/v1")
|
||
|
|
unknown := discoverNative(t, "support/summary", "--session-id=session")
|
||
|
|
if unknown.Complete || len(unknown.Issues) == 0 {
|
||
|
|
t.Fatalf("unknown condition claimed complete: %+v", unknown)
|
||
|
|
}
|
||
|
|
for _, input := range unknown.Inputs {
|
||
|
|
if input.Name == "glossary" && input.Kind != promptkit.RequirementConditional {
|
||
|
|
t.Fatalf("unknown branch: %+v", input)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
known := discoverNative(t, "support/summary", "--var=mode=include", "--session-id=session")
|
||
|
|
if !known.Complete {
|
||
|
|
t.Fatalf("known branch incomplete: %+v", known)
|
||
|
|
}
|
||
|
|
for _, input := range known.Inputs {
|
||
|
|
if input.Name == "glossary" && input.Kind != promptkit.RequirementRequired {
|
||
|
|
t.Fatalf("known branch: %+v", input)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
dynamic := discoverNative(t, "support/dynamic")
|
||
|
|
if dynamic.Complete || len(dynamic.Issues) == 0 {
|
||
|
|
t.Fatalf("dynamic name not partial: %+v", dynamic)
|
||
|
|
}
|
||
|
|
resolved := discoverNative(t, "support/dynamic", "--var=source=exact,name")
|
||
|
|
if !resolved.Complete || len(resolved.Inputs) != 1 || resolved.Inputs[0].Name != "exact,name" || resolved.Inputs[0].Declared {
|
||
|
|
t.Fatalf("resolved identifier: %+v", resolved)
|
||
|
|
}
|
||
|
|
unsupported := discoverNative(t, "support/unsupported")
|
||
|
|
var found bool
|
||
|
|
for _, issue := range unsupported.Issues {
|
||
|
|
found = found || issue.Code == promptkit.RequirementIssueUnsupportedConstruct
|
||
|
|
}
|
||
|
|
if unsupported.Complete || !found {
|
||
|
|
t.Fatalf("unsupported analysis not preserved: %+v", unsupported)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRequirementsSelectionUsesNativeNamespacesAndFailures(t *testing.T) {
|
||
|
|
nativePromptFixture(t, "http://127.0.0.1:1/v1")
|
||
|
|
before := discoverNative(t, "summary", "--prompt-namespace=support", "--var=mode=", "--session-id=session")
|
||
|
|
after := discoverNative(t, "support/summary", "--prompt-namespace=review", "--profile=writer", "--profile-namespace=review", "--var=mode=", "--session-id=session")
|
||
|
|
if after.PromptID != "support/summary" || after.ProfileID != "review/writer" || !reflect.DeepEqual(before.Inputs, after.Inputs) {
|
||
|
|
t.Fatalf("identity/definition changed: before=%+v after=%+v", before, after)
|
||
|
|
}
|
||
|
|
other := discoverNative(t, "review/summary")
|
||
|
|
if other.PromptID != "review/summary" || other.ProfileID != "review/writer" || len(other.Inputs) != 1 || other.Inputs[0].Name != "memo" {
|
||
|
|
t.Fatalf("other definition: %+v", other)
|
||
|
|
}
|
||
|
|
for _, args := range [][]string{
|
||
|
|
{"promptkit", "requirements", "show", "missing", "--config=app.yml"},
|
||
|
|
{"promptkit", "requirements", "show", "support/summary", "--config=app.yml", "--profile=missing"},
|
||
|
|
} {
|
||
|
|
outcome, output, diagnostics := executeNative(t, args...)
|
||
|
|
if outcome.Status == 0 || output != "" || diagnostics == "" {
|
||
|
|
t.Fatalf("selection failure: %+v %q %q", outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestNativeInventoriesInspectionAndPreparationDoNotGenerate(t *testing.T) {
|
||
|
|
var calls atomic.Int64
|
||
|
|
var fail atomic.Bool
|
||
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
|
|
calls.Add(1)
|
||
|
|
w.Header().Set("Content-Type", "application/json")
|
||
|
|
if fail.Load() {
|
||
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||
|
|
fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"controlled"}}]}`)
|
||
|
|
}))
|
||
|
|
defer server.Close()
|
||
|
|
nativePromptFixture(t, server.URL+"/v1")
|
||
|
|
for _, args := range [][]string{
|
||
|
|
{"promptkit", "list", "prompts", "--all-namespaces"},
|
||
|
|
{"promptkit", "list", "profiles", "--namespace=support"},
|
||
|
|
{"promptkit", "list", "backends", "--namespace=support"},
|
||
|
|
{"promptkit", "list", "namespaces"},
|
||
|
|
{"promptkit", "list", "collections"},
|
||
|
|
{"promptkit", "show", "prompt", "support/summary"},
|
||
|
|
{"promptkit", "show", "profile", "support/writer"},
|
||
|
|
{"promptkit", "show", "backend", "support/local"},
|
||
|
|
} {
|
||
|
|
args = append(args, "--config=app.yml", "--format=json")
|
||
|
|
outcome, output, diagnostics := executeNative(t, args...)
|
||
|
|
if outcome.Status != 0 || output == "" {
|
||
|
|
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
var value any
|
||
|
|
if err := json.Unmarshal([]byte(output), &value); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if args[1] == "show" && args[2] == "prompt" && !strings.Contains(output, "template-canary") {
|
||
|
|
t.Fatalf("native prompt content omitted: %s", output)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
engine, err := loadEngine(context.Background(), ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "app.yml", Set: true}}, applicationInput{}, nil)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
nativePrompts, err := engine.ListPrompts(context.Background())
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
outcome, output, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--config=app.yml", "--format=json")
|
||
|
|
var listed []promptkit.PromptIdentity
|
||
|
|
if err := json.Unmarshal([]byte(output), &listed); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if outcome.Status != 0 || !reflect.DeepEqual(listed, nativePrompts) {
|
||
|
|
t.Fatalf("native default listing changed: %+v %q %q", outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
outcome, output, diagnostics = executeNative(t, "promptkit", "list", "prompts", "--namespace=review", "--config=app.yml", "--format=json")
|
||
|
|
if outcome.Status != 0 || strings.Contains(output, "support/summary") || !strings.Contains(output, "review/summary") {
|
||
|
|
t.Fatalf("namespace filter lost: %+v %q %q", outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
writeNativeFile(t, "transcript.md", "artifact-payload-canary")
|
||
|
|
flags := []string{"summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--session-id=session", "--var=mode=", "--var=title=Title", "--input-file=transcript=transcript.md", "--input=unlisted=literal"}
|
||
|
|
for _, command := range [][]string{{"promptkit", "requirements", "validate"}, {"promptkit", "show", "prepared"}} {
|
||
|
|
args := append(append([]string{}, command...), flags...)
|
||
|
|
outcome, output, diagnostics := executeNative(t, append(args, "--format=json")...)
|
||
|
|
if outcome.Status != 0 {
|
||
|
|
t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
if command[1] == "show" && (!strings.Contains(output, "artifact-payload-canary") || !strings.Contains(output, "support/writer") || !strings.Contains(output, "support/summary")) {
|
||
|
|
t.Fatalf("native prepared content/context lost: %s", output)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for _, format := range []string{"", "text", "yaml"} {
|
||
|
|
args := append([]string{"promptkit", "show", "prepared"}, flags...)
|
||
|
|
if format != "" {
|
||
|
|
args = append(args, "--format="+format)
|
||
|
|
}
|
||
|
|
outcome, output, diagnostics := executeNative(t, args...)
|
||
|
|
if outcome.Status != 0 || !strings.Contains(output, "artifact-payload-canary") {
|
||
|
|
t.Fatalf("native preview %q: %+v %q %q", format, outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if calls.Load() != 0 {
|
||
|
|
t.Fatal("inspection/preparation contacted provider")
|
||
|
|
}
|
||
|
|
// Smoke tests are the explicit provider-effect boundary. Native findings,
|
||
|
|
// not an application readiness policy, determine what the report says.
|
||
|
|
outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=writer", "--profile-namespace=support", "--format=json")
|
||
|
|
if outcome.Status != 0 || calls.Load() != 1 || !strings.Contains(output, "text_inference") {
|
||
|
|
t.Fatalf("controlled smoke: %+v %q %q calls=%d", outcome, output, diagnostics, calls.Load())
|
||
|
|
}
|
||
|
|
fail.Store(true)
|
||
|
|
outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=support/writer", "--format=json")
|
||
|
|
var findings struct {
|
||
|
|
Completed bool `json:"completed"`
|
||
|
|
Failure string `json:"failure"`
|
||
|
|
}
|
||
|
|
if err := json.Unmarshal([]byte(output), &findings); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if outcome.Status != 0 || !findings.Completed || findings.Failure == "" || strings.Contains(output+diagnostics, "provider-private-canary") {
|
||
|
|
t.Fatalf("native unsuccessful finding changed: %+v %q %q", outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
writeNativeFile(t, "invalid-app.yml", "server:\n max_request_bytes: -1\n")
|
||
|
|
outcome, output, diagnostics = executeNative(t, "promptkit", "requirements", "show", "support/summary", "--config=invalid-app.yml", "--quiet")
|
||
|
|
if outcome.Kind != commandkit.OutcomeUsage || output != "" || !strings.Contains(diagnostics, "Application configuration") {
|
||
|
|
t.Fatalf("acquisition evidence: %+v %q %q", outcome, output, diagnostics)
|
||
|
|
}
|
||
|
|
}
|