Files
scriptorium/internal/adapter/cli/native_run_test.go

235 lines
12 KiB
Go

package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
"go.yaml.in/yaml/v4"
)
func TestPositionalRunMapsNativeBindingsAndNamespaces(t *testing.T) {
requests := make(chan map[string]any, 1)
const content = " \x00<artifact>&\n尾-no-newline"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var request map[string]any
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
}
request["authorization"] = r.Header.Get("Authorization")
select {
case requests <- request:
default:
t.Error("unexpected duplicate provider call")
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}}}})
}))
defer server.Close()
nativePromptFixture(t, "http://127.0.0.1:1/v1")
t.Setenv("SCRIPTORIUM_CONTROLLED_KEY", "synthetic-credential-canary")
writeNativeFile(t, "note,part=1.txt", "file,transcript=body")
args := []string{"run", "summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--backend-namespace=review", "--session-id=caller-session",
"--input-file=transcript=note,part=1.txt", "--input-text=transcript=first,text=1", "--input-file=transcript=note,part=1.txt", "--input-file=unlisted=missing.txt", "--input=unlisted=last,literal=2", "--var=mode=", "--var=title=Title,with=equals",
"--endpoint=" + server.URL + "/v1", "--model=chosen-model", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0", "--reasoning-effort=", "--service-tier=priority", `--extra-params={"seed":0,"test_payload":[0,"x"]}`, "--api-key-env=SCRIPTORIUM_CONTROLLED_KEY", "--transient-retries=0", "--run-timeout=5s", "--format=json"}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 || strings.Contains(output+diagnostics, "synthetic-credential-canary") {
t.Fatalf("run: %+v %q %q", outcome, output, diagnostics)
}
var result promptkit.RunResult
if err := json.Unmarshal([]byte(output), &result); err != nil {
t.Fatal(err)
}
if string(result.Artifact.Body) != content || result.PromptID != "support/summary" || result.SelectedProfileID != "support/writer" || result.SelectedBackendID != "support/local" || result.SessionID != "caller-session" {
t.Fatalf("native result/context: %+v", result)
}
params := result.EffectiveModelParams
if params.Model != "chosen-model" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 || params.TimeoutSeconds != 0 || params.ReasoningEffort != "" || params.ServiceTier != "priority" {
t.Fatalf("native explicit overrides: %+v", params)
}
select {
case request := <-requests:
messages := request["messages"].([]any)
message := messages[0].(map[string]any)["content"].(string)
for _, want := range []string{"file,transcript=body", "last,literal=2", "Title,with=equals"} {
if !strings.Contains(message, want) {
t.Fatalf("lost literal/mapping %q: %s", want, message)
}
}
for _, key := range []string{"temperature", "max_tokens", "top_p", "seed"} {
if value, exists := request[key]; !exists || value != float64(0) {
t.Fatalf("explicit zero %s lost: %+v", key, request)
}
}
if request["model"] != "chosen-model" || request["service_tier"] != "priority" || request["authorization"] != "Bearer synthetic-credential-canary" || request["stale"] != nil || request["reasoning_effort"] != nil {
t.Fatalf("provider overrides: %+v", request)
}
default:
t.Fatal("provider was not called")
}
}
func TestRunDeliversRawAndNativeStructuredResults(t *testing.T) {
const content = " <raw>&\x00\nno-final-newline"
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}})
}))
defer server.Close()
nativePromptFixture(t, server.URL+"/v1")
flags := []string{"support/summary", "--config=app.yml", "--llm-profile=support/writer", "--llm-session-id=session", "--llm-input=transcript=text", "--llm-input-text=unlisted=literal", "--llm-var=mode=", "--llm-var=title=Title"}
for _, format := range []string{"", "text", "json", "yaml"} {
args := append([]string{"run"}, flags...)
if format != "" {
args = append(args, "--format="+format)
}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 0 {
t.Fatalf("%s: %+v %q %q", format, outcome, output, diagnostics)
}
if format == "" || format == "text" {
if output != content || diagnostics != "" {
t.Fatalf("raw output decorated: %q %q", output, diagnostics)
}
} else {
var report map[string]any
if err := yaml.Unmarshal([]byte(output), &report); err != nil {
t.Fatal(err)
}
if report["prompt_id"] != "support/summary" || report["run_id"] == nil || report["artifact"] == nil || report["result"] != nil || report["schema_version"] != nil {
t.Fatalf("non-native result: %+v", report)
}
}
}
outcome, output, diagnostics := executeNative(t, append(append([]string{"run"}, flags...), "--verbose")...)
if outcome.Status != 0 || output != content || diagnostics == "" {
t.Fatalf("verbose streams: %+v %q %q", outcome, output, diagnostics)
}
if calls.Load() != 5 {
t.Fatalf("generation dispatches=%d", calls.Load())
}
// The same selected prompt/profile/context used above is valid through the
// native preview and requirements paths (Narratio's consistency scenario).
discovered := discoverNative(t, "support/summary", "--profile=support/writer", "--session-id=session", "--var=mode=")
if discovered.PromptID != "support/summary" || discovered.ProfileID != "support/writer" {
t.Fatal(discovered)
}
}
func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing.T) {
var fail atomic.Bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if fail.Load() {
w.WriteHeader(401)
fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`)
return
}
fmt.Fprint(w, `{"choices":[{"message":{"content":"not-json"}}]}`)
}))
defer server.Close()
nativePromptFixture(t, server.URL+"/v1")
writeNativeFile(t, "invalid-content.yml", "profiles:\n - id: local\n endpoint: "+server.URL+"/v1\n model: controlled\nprompts:\n - id: json\n default_profile: local\n messages:\n - role: user\n content: Return JSON\n output:\n format: json\n validation_mode: json\n")
writeNativeFile(t, "json-app.yml", "promptkit:\n configuration: invalid-content.yml\ntransient_retries: 0\n")
args := []string{"run", "json", "--config=json-app.yml"}
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 4 || outcome.Kind != commandkit.OutcomeDomain || outcome.Domain != "validation_failed" || output != "not-json" || diagnostics == "" {
t.Fatalf("completed invalid content: %+v %q %q", outcome, output, diagnostics)
}
outcome, output, diagnostics = executeNative(t, append(args, "--format=json")...)
var result promptkit.RunResult
if err := json.Unmarshal([]byte(output), &result); err != nil {
t.Fatal(err)
}
if outcome.Status != 4 || result.Validation.Status != promptkit.ValidationFailed || string(result.Artifact.Body) != "not-json" {
t.Fatalf("invalid result lost: %+v %+v", outcome, result)
}
fail.Store(true)
outcome, output, diagnostics = executeNative(t, args...)
var generation *promptkit.GenerationError
if outcome.Status != 1 || output != "" || !errors.Is(outcome.Cause, promptkit.ErrLLMGenerate) || !errors.As(outcome.Cause, &generation) || diagnostics == "" || strings.Contains(diagnostics, "provider-private-canary") {
t.Fatalf("operational failure: %+v %q %q", outcome, output, diagnostics)
}
}
func TestRunUsageAndConfigurationFailures(t *testing.T) {
t.Chdir(t.TempDir())
for _, args := range [][]string{
{"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"},
{"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"},
{"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"},
{"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"},
} {
outcome, _, _ := executeNative(t, args...)
if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {
t.Fatalf("%v: %+v", args, outcome)
}
}
writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n")
outcome, output, diagnostics := executeNative(t, "run", "one", "--config=invalid.yml")
if outcome.Status != 2 || output != "" || !strings.Contains(diagnostics, "validate.error") {
t.Fatalf("invalid config: %+v %q %q", outcome, output, diagnostics)
}
outcome, output, diagnostics = executeNative(t, "run", "one", "--config=absent.yml")
if outcome.Status != 1 || output != "" || diagnostics == "" {
t.Fatalf("missing config: %+v %q %q", outcome, output, diagnostics)
}
}
func TestTypedRequestFailureMapping(t *testing.T) {
_, invalid := configkit.Load(configkit.Bytes("invalid", []byte("[\n")), configkit.Options[map[string]any]{})
for _, test := range []struct {
err error
status int
}{
{fmt.Errorf("wrapped: %w", invalid), 2}, {promptkit.ErrInvalidConfig, 1},
{errors.Join(invalid, context.Canceled), 1}, {errors.Join(invalid, promptkit.ErrOperationTimeout, context.DeadlineExceeded), 1},
{promptkit.ErrCapacityExceeded, 1}, {promptkit.ErrValidation, 1}, {promptkit.ErrInvalidRequest, 1},
{promptkit.ErrPromptNotFound, 1}, {promptkit.ErrLLMGenerate, 1},
} {
outcome := requestFailure(test.err, commandkit.Explanation{})
if outcome.Status != test.status || !errors.Is(outcome.Cause, test.err) {
t.Fatalf("%v: %+v", test.err, outcome)
}
}
if outcome := completedRunOutcome(nil); outcome.Status != 1 || outcome.Cause == nil {
t.Fatalf("nil successful result accepted: %+v", outcome)
}
// Mapping a duration adds no local positivity policy or second timer.
for _, budget := range []time.Duration{-time.Second, 0, time.Second} {
request := requestFromRunInput(runInput{OperationTimeout: budget})
if request.OperationTimeout != budget {
t.Fatal("operation budget changed", budget)
}
}
}
func TestRunDiagnosticQueueKeepsArtifactOnStdout(t *testing.T) {
app, err := commandkit.New(commandkit.Identity{Name: "reporting"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
queueDiagnostics(inv.Reporter(), []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityInfo, "acquisition-info"), commandkit.MessageDiagnostic(commandkit.SeverityWarning, "acquisition-warning")})
_ = inv.Reporter().Report(commandkit.Report{Value: &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("raw")}}, TextTemplate: artifactTextTemplate})
return commandkit.Success()
}}))
if err != nil {
t.Fatal(err)
}
var output, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"run", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Status != 0 || output.String() != "raw" || !strings.Contains(diagnostics.String(), "acquisition-info") || !strings.Contains(diagnostics.String(), "acquisition-warning") {
t.Fatalf("streams: %+v %q %q", outcome, output.String(), diagnostics.String())
}
}