197 lines
7.7 KiB
Go
197 lines
7.7 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
commandkit "go.maximumdirect.net/commandkit"
|
|
pk "go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestNativeConfigurationAndResourceCommands(t *testing.T) {
|
|
dir := t.TempDir()
|
|
file := filepath.Join(dir, "config.yml")
|
|
if err := os.WriteFile(file, []byte("version: 3\ncontext:\n description: example\n"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
app, err := buildApplication()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, tc := range []struct {
|
|
args []string
|
|
contains string
|
|
}{
|
|
{[]string{"--help"}, "audita"},
|
|
{[]string{"version"}, "audita"},
|
|
{[]string{"process", "--help"}, "transcript"},
|
|
{[]string{"config", "show", "--scope", "app", "--config", file}, "example"},
|
|
{[]string{"promptkit", "list", "prompts", "--config", file}, "audita/modules.grammar.proposal"},
|
|
} {
|
|
var out, stderr bytes.Buffer
|
|
result := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: &out, Err: &stderr})
|
|
if result.Status != 0 || !strings.Contains(out.String(), tc.contains) {
|
|
t.Errorf("%v: status=%d out=%q err=%q", tc.args, result.Status, out.String(), stderr.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeConfigurationSelectionsAndComparison(t *testing.T) {
|
|
dir := t.TempDir()
|
|
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
|
|
for name, data := range map[string]string{first: "context:\n description: before\n", second: "context:\n description: after\n"} {
|
|
if err := os.WriteFile(name, []byte(data), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
app, err := buildApplication()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out, stderr bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--scope=app", "--config", first, "--against-config", second, "--set=/context/description=\"left\"", "--against-set=/context/description=\"right\""}, commandkit.Streams{Out: &out, Err: &stderr})
|
|
if outcome.Status != 3 || !strings.Contains(out.String(), "left") || !strings.Contains(out.String(), "right") {
|
|
t.Fatalf("status=%d output=%q err=%q", outcome.Status, out.String(), stderr.String())
|
|
}
|
|
out.Reset()
|
|
stderr.Reset()
|
|
outcome = app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config", filepath.Join(dir, "missing.yml")}, commandkit.Streams{Out: &out, Err: &stderr})
|
|
if outcome.Status == 0 {
|
|
t.Fatalf("missing file accepted: %s", out.String())
|
|
}
|
|
}
|
|
|
|
func TestExplicitEmptyConfigurationPathFails(t *testing.T) {
|
|
app, err := buildApplication()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out, stderr bytes.Buffer
|
|
result := app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config="}, commandkit.Streams{Out: &out, Err: &stderr})
|
|
if result.Status == 0 {
|
|
t.Fatal("explicit empty path accepted")
|
|
}
|
|
}
|
|
|
|
func TestRequestInspectionAndExplicitSmokeUseSameSources(t *testing.T) {
|
|
dir := t.TempDir()
|
|
file := filepath.Join(dir, "config.yml")
|
|
if err := os.WriteFile(file, []byte("version: 3\n"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resources := `backends:
|
|
- id: local
|
|
endpoint: https://example.invalid/v1
|
|
smoke_tests:
|
|
default_profile: fixture
|
|
authentication:
|
|
kind: none
|
|
profiles:
|
|
- id: fixture
|
|
backend: local
|
|
model: sample
|
|
collections:
|
|
audita:
|
|
prompts:
|
|
- id: modules.grammar.proposal
|
|
messages:
|
|
- role: user
|
|
content: 'Replacement {{.HardeningPolicy}} {{.SectionJSON}}'
|
|
output:
|
|
format: text
|
|
validation_mode: none
|
|
`
|
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := 0
|
|
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
calls++
|
|
body := `{}`
|
|
if req.Method == http.MethodPost {
|
|
requestBody, err := io.ReadAll(req.Body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
content := "PROMPTKIT_OK"
|
|
if strings.Contains(string(requestBody), "response_format") {
|
|
content = `{"status":"ok"}`
|
|
}
|
|
encoded, err := json.Marshal(content)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body = `{"model":"sample","choices":[{"message":{"content":` + string(encoded) + `}}]}`
|
|
}
|
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
|
|
})}
|
|
app, err := buildApplicationWithClient(client)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
execute := func(args ...string) (commandkit.Outcome, string, string) {
|
|
t.Helper()
|
|
var out, stderr bytes.Buffer
|
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
|
return result, out.String(), stderr.String()
|
|
}
|
|
common := []string{"--config", file, "--llm-profile", "fixture"}
|
|
args := append([]string{"promptkit", "show", "prepared", "audita/modules.grammar.proposal"}, common...)
|
|
args = append(args, "--llm-var", "SectionJSON=[]", "--llm-var", "HardeningPolicy=custom-instruction")
|
|
outcome, out, stderr := execute(args...)
|
|
if outcome.Status != 0 || !strings.Contains(out, "Replacement custom-instruction") {
|
|
t.Fatalf("preview status=%d out=%q err=%q", outcome.Status, out, stderr)
|
|
}
|
|
args = append([]string{"promptkit", "requirements", "validate", "audita/validators.editorial_review"}, common...)
|
|
args = append(args, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
|
|
outcome, out, stderr = execute(args...)
|
|
if outcome.Status != 0 {
|
|
t.Fatalf("validator status=%d out=%q err=%q", outcome.Status, out, stderr)
|
|
}
|
|
validatorPreview := append([]string{"promptkit", "show", "prepared", "audita/validators.editorial_review"}, common...)
|
|
validatorPreview = append(validatorPreview, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
|
|
outcome, out, stderr = execute(validatorPreview...)
|
|
if outcome.Status != 0 || !strings.Contains(out, "teh cat") {
|
|
t.Fatalf("validator preview status=%d out=%q err=%q", outcome.Status, out, stderr)
|
|
}
|
|
if calls != 0 {
|
|
t.Fatalf("inspection made %d provider calls", calls)
|
|
}
|
|
outcome, out, stderr = execute("promptkit", "test", "reachability", "--config", file, "--llm-backend", "local")
|
|
if outcome.Status != 0 || calls == 0 {
|
|
t.Fatalf("smoke status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
|
|
}
|
|
previous := calls
|
|
outcome, out, stderr = execute("promptkit", "test", "all", "--config", file, "--llm-backend", "local", "--llm-profile", "fixture", "--format=json")
|
|
if outcome.Status != 0 || calls <= previous || !strings.Contains(out, "structured_output") || !strings.Contains(out, "text_inference") {
|
|
t.Fatalf("aggregate status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
|
|
}
|
|
}
|
|
|
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
|
|
|
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
|
|
|
func TestRequestDefaultsRespectNamespaceAndCallerVariables(t *testing.T) {
|
|
request := pk.RunRequest{PromptID: "modules.grammar.proposal", Vars: map[string]string{"HardeningPolicy": "caller"}}
|
|
result := applicationPromptRequest(request, &pk.Namespaces{Prompt: "audita"})
|
|
if result.Vars["HardeningPolicy"] != "caller" || result.Vars["ProposalContract"] == "" {
|
|
t.Fatalf("variables=%v", result.Vars)
|
|
}
|
|
if _, changed := request.Vars["ProposalContract"]; changed {
|
|
t.Fatal("input variables mutated")
|
|
}
|
|
generic := pk.RunRequest{PromptID: "generic", Vars: map[string]string{"x": "y"}}
|
|
result = applicationPromptRequest(generic, nil)
|
|
if len(result.Vars) != 1 || result.Vars["x"] != "y" {
|
|
t.Fatalf("generic request changed: %v", result.Vars)
|
|
}
|
|
}
|