Verify the complete native CLI cutover and record roadmap completion
This commit is contained in:
@@ -3,11 +3,15 @@ package cli
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
|
||||
@@ -91,3 +95,102 @@ func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintainedCollectionWorkflow(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Chdir(root)
|
||||
requests := make(chan []promptkit.RenderedMessage, 2)
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
|
||||
var body struct {
|
||||
Messages []promptkit.RenderedMessage `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(request.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
select {
|
||||
case requests <- body.Messages:
|
||||
default:
|
||||
t.Error("unexpected provider dispatch")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"choices":[{"message":{"content":"collection result"}}]}`)
|
||||
}))
|
||||
defer provider.Close()
|
||||
const config = "examples/config.discovery.yml"
|
||||
outcome, inventory, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--all-namespaces", "--config="+config, "--format=json")
|
||||
if outcome.Status != 0 || !strings.Contains(inventory, "support/answer") || !strings.Contains(inventory, "support/summarize") || !strings.Contains(inventory, "review/answer") {
|
||||
t.Fatalf("collection discovery: %+v %q %q", outcome, inventory, diagnostics)
|
||||
}
|
||||
inputPath := filepath.Join(t.TempDir(), "future-transcript.md")
|
||||
for _, namespace := range []string{"support", "review"} {
|
||||
profile := "fast"
|
||||
if namespace == "review" {
|
||||
profile = "base"
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, "promptkit", "requirements", "show", "answer", "--config="+config, "--prompt-namespace="+namespace, "--profile="+profile, "--profile-namespace="+namespace, "--format=json")
|
||||
var report consumerRequirements
|
||||
if err := json.Unmarshal([]byte(output), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Status != 0 || !report.Complete || report.PromptID != namespace+"/answer" || report.ProfileID != namespace+"/"+profile {
|
||||
t.Fatalf("requirements: %+v %+v %q", outcome, report, diagnostics)
|
||||
}
|
||||
if namespace == "support" && (len(report.Inputs) != 1 || report.Inputs[0].Name != "transcript" || report.Inputs[0].Kind != promptkit.RequirementRequired) {
|
||||
t.Fatalf("required binding: %+v", report)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(inputPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("discovery created input artifact: %v", err)
|
||||
}
|
||||
writeNativeFile(t, inputPath, "generated collection transcript")
|
||||
for _, namespace := range []string{"support", "review"} {
|
||||
profile := "fast"
|
||||
if namespace == "review" {
|
||||
profile = "base"
|
||||
}
|
||||
flags := []string{"answer", "--config=" + config, "--prompt-namespace=" + namespace, "--profile=" + profile, "--profile-namespace=" + namespace, "--endpoint=" + provider.URL + "/v1", "--model=controlled", "--timeout-seconds=0", "--transient-retries=0", "--session-id=collection-session", "--format=json"}
|
||||
if namespace == "support" {
|
||||
flags = append(flags, "--input-file=transcript="+inputPath)
|
||||
}
|
||||
outcome, _, diagnostics := executeNative(t, append([]string{"promptkit", "requirements", "validate"}, flags...)...)
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("validation: %+v %q", outcome, diagnostics)
|
||||
}
|
||||
outcome, output, diagnostics := executeNative(t, append([]string{"promptkit", "show", "prepared"}, flags...)...)
|
||||
var prepared promptkit.PreparedRun
|
||||
if err := json.Unmarshal([]byte(output), &prepared); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("preview: %+v %q", outcome, diagnostics)
|
||||
}
|
||||
outcome, output, diagnostics = executeNative(t, append([]string{"run"}, flags...)...)
|
||||
var result promptkit.RunResult
|
||||
if err := json.Unmarshal([]byte(output), &result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outcome.Status != 0 || result.PromptID != prepared.PromptID || result.SelectedProfileID != prepared.SelectedProfileID || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(result.InputHashes, prepared.InputHashes) || string(result.Artifact.Body) != "collection result" {
|
||||
t.Fatalf("workflow selection/context diverged: prepared=%+v result=%+v outcome=%+v %q", prepared, result, outcome, diagnostics)
|
||||
}
|
||||
select {
|
||||
case messages := <-requests:
|
||||
if !reflect.DeepEqual(messages, prepared.Messages) {
|
||||
t.Fatalf("provider saw different messages: %+v %+v", messages, prepared.Messages)
|
||||
}
|
||||
default:
|
||||
t.Fatal("generation did not reach controlled provider")
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{
|
||||
{"config", "diff", "--scope=app", "--config=" + config, "--against-config=" + config, `--set=/promptkit/schema_dir="left"`, `--against-set=/promptkit/schema_dir="right"`, "--check"},
|
||||
{"config", "diff", "--scope=promptkit", "--config=" + config, "--against-config=examples/config.yml", "--check"},
|
||||
} {
|
||||
outcome, output, diagnostics := executeNative(t, args...)
|
||||
if outcome.Status != 3 || output == "" {
|
||||
t.Fatalf("independent configuration sides: %+v %q %q", outcome, output, diagnostics)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user