Files
scriptorium/internal/adapter/cli/lifecycle_test.go

236 lines
8.4 KiB
Go

package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func TestRunCancellationAndNativeTimeoutLayers(t *testing.T) {
for _, test := range []struct {
name, operation, attempt string
cancelParent, localTimeout bool
parentDeadline bool
}{
{"parent cancellation", "10s", "0", true, false, false},
{"operation timeout", "500ms", "0", false, true, false},
{"zero operation budget", "0", "0", true, false, false},
{"negative operation budget", "-1s", "0", true, false, false},
{"attempt timeout", "0", "1", false, false, false},
{"earlier parent deadline", "10s", "0", false, false, true},
} {
t.Run(test.name, func(t *testing.T) {
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
close(started)
select {
case <-r.Context().Done():
close(stopped)
case <-release:
}
}))
defer provider.Close()
defer close(release)
nativePromptFixture(t, provider.URL+"/v1")
var ctx context.Context
var cancel context.CancelFunc
if test.parentDeadline {
ctx, cancel = context.WithTimeout(context.Background(), 500*time.Millisecond)
} else {
ctx, cancel = context.WithCancel(context.Background())
}
defer cancel()
var output, diagnostics bytes.Buffer
finished := make(chan commandkit.Outcome, 1)
args := append(deliveryArgs(), "--run-timeout="+test.operation, "--timeout-seconds="+test.attempt)
go func() {
finished <- Execute(ctx, commandkit.Identity{Name: "scriptorium"}, args, &output, &diagnostics)
}()
select {
case <-started:
case outcome := <-finished:
t.Fatalf("ended before provider dispatch: %+v %q", outcome, diagnostics.String())
case <-time.After(5 * time.Second):
t.Fatal("provider did not start")
}
if test.cancelParent {
cancel()
}
var outcome commandkit.Outcome
select {
case outcome = <-finished:
case <-time.After(5 * time.Second):
t.Fatal("run did not stop")
}
if outcome.Status != 1 || output.Len() != 0 || diagnostics.Len() == 0 || errors.Is(outcome.Cause, promptkit.ErrOperationTimeout) != test.localTimeout {
t.Fatalf("timeout/cancellation: %+v %q %q", outcome, output.String(), diagnostics.String())
}
if test.cancelParent && !errors.Is(outcome.Cause, context.Canceled) {
t.Fatalf("parent cause lost: %+v", outcome)
}
if !test.cancelParent && !errors.Is(outcome.Cause, context.DeadlineExceeded) {
t.Fatalf("deadline cause lost: %+v", outcome)
}
if !test.cancelParent && !test.parentDeadline && ctx.Err() != nil {
t.Fatal("engine deadline changed caller context")
}
select {
case <-stopped:
case <-time.After(5 * time.Second):
t.Fatal("provider request was not canceled")
}
})
}
}
func TestServeStartupHTTPAndCancellation(t *testing.T) {
var block atomic.Bool
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
if block.Load() {
close(started)
select {
case <-r.Context().Done():
close(stopped)
case <-release:
}
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"choices":[{"message":{"content":"served"}}]}`)
}))
defer provider.Close()
defer close(release)
nativePromptFixture(t, provider.URL+"/v1")
writeNativeFile(t, "artifacts/transcript.md", "file from restricted root")
ready := make(chan string, 1)
listen := func(ctx context.Context, network, addr string) (net.Listener, error) {
listener, err := (&net.ListenConfig{}).Listen(ctx, network, addr)
if err == nil {
ready <- listener.Addr().String()
}
return listener, err
}
app, err := commandkit.New(commandkit.Identity{Name: "scriptorium"}, commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Flags: serveFlags(), Run: func(inv commandkit.Invocation, input serveInput) commandkit.Outcome {
return serveWithListener(inv, input, listen)
}}))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var output, diagnostics bytes.Buffer
finished := make(chan commandkit.Outcome, 1)
go func() {
finished <- app.Execute(ctx, []string{"serve", "--config=app.yml", "--addr=127.0.0.1:0", "--artifact-root=artifacts", "--max-request-bytes=0", "--max-artifact-bytes=0", "--max-response-bytes=0", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
}()
var addr string
select {
case addr = <-ready:
case outcome := <-finished:
t.Fatalf("startup: %+v %q", outcome, diagnostics.String())
case <-time.After(5 * time.Second):
t.Fatal("server did not listen")
}
client := &http.Client{Timeout: 5 * time.Second}
defer client.CloseIdleConnections()
const payload = `{"prompt_id":"support/summary","profile_id":"support/writer","session_id":"server-session","vars":{"mode":"","title":"Title"},"inputs":{"transcript":{"type":"file","uri":"transcript.md"},"unlisted":{"type":"inline","body":"literal"}},"model":{"timeout_seconds":0},"include_execution_report":true}`
response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload))
if err != nil {
t.Fatal(err)
}
var value map[string]any
decodeErr := json.NewDecoder(response.Body).Decode(&value)
response.Body.Close()
if response.StatusCode != 200 || decodeErr != nil || value["execution"] == nil {
t.Fatalf("HTTP response: %d %+v %v", response.StatusCode, value, decodeErr)
}
metadata := value["metadata"].(map[string]any)
if metadata["selected_profile_id"] != "support/writer" || metadata["session_id"] != "server-session" {
t.Fatalf("HTTP contract changed: %+v", value)
}
block.Store(true)
requestFinished := make(chan error, 1)
go func() {
response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload))
if response != nil {
response.Body.Close()
}
requestFinished <- err
}()
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("active HTTP request did not reach engine")
}
cancel()
select {
case outcome := <-finished:
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("shutdown: %+v %q %q", outcome, output.String(), diagnostics.String())
}
case <-time.After(5 * time.Second):
t.Fatal("HTTP server did not close")
}
select {
case <-requestFinished:
case <-time.After(5 * time.Second):
t.Fatal("active HTTP connection remained open")
}
select {
case <-stopped:
case <-time.After(5 * time.Second):
t.Fatal("active engine work was not canceled")
}
if connection, err := net.DialTimeout("tcp", addr, time.Second); err == nil {
connection.Close()
t.Fatal("listener remained open")
}
}
func TestServeListenAndConfigurationFailures(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
for _, args := range [][]string{
{"serve", "--config=app.yml", "--addr=invalid:address:syntax"},
{"serve", "--config=absent.yml"},
} {
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 1 || outcome.Cause == nil || output != "" || diagnostics == "" {
t.Fatalf("serve failure: %+v %q %q", outcome, output, diagnostics)
}
}
writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n")
outcome, _, diagnostics := executeNative(t, "serve", "--config=invalid.yml")
if outcome.Status != 2 || !strings.Contains(diagnostics, "validate.error") {
t.Fatalf("serve configuration classification: %+v %q", outcome, diagnostics)
}
writeNativeFile(t, "bad-framework.yml", "profiles: [\n")
writeNativeFile(t, "bad-framework-app.yml", "promptkit:\n configuration: bad-framework.yml\n")
for _, command := range []string{"run", "serve"} {
args := []string{command}
if command == "run" {
args = append(args, "any")
}
args = append(args, "--config=bad-framework-app.yml")
outcome, _, diagnostics := executeNative(t, args...)
if outcome.Status != 2 || !strings.Contains(diagnostics, "PromptKit configuration") || !strings.Contains(diagnostics, "yaml.invalid") {
t.Fatalf("framework diagnostics: %+v %q", outcome, diagnostics)
}
}
}