package cli import ( "bytes" "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" "go.yaml.in/yaml/v4" ) func TestPositionalRunMapsNativeBindingsAndNamespaces(t *testing.T) { requests := make(chan map[string]any, 1) const content = " \x00&\n尾-no-newline" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var request map[string]any if err := json.NewDecoder(r.Body).Decode(&request); err != nil { t.Error(err) } request["authorization"] = r.Header.Get("Authorization") select { case requests <- request: default: t.Error("unexpected duplicate provider call") } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}}}}) })) defer server.Close() nativePromptFixture(t, "http://127.0.0.1:1/v1") t.Setenv("SCRIPTORIUM_CONTROLLED_KEY", "synthetic-credential-canary") writeNativeFile(t, "note,part=1.txt", "file,transcript=body") args := []string{"run", "summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--backend-namespace=review", "--session-id=caller-session", "--input-file=transcript=note,part=1.txt", "--input-text=transcript=first,text=1", "--input-file=transcript=note,part=1.txt", "--input-file=unlisted=missing.txt", "--input=unlisted=last,literal=2", "--var=mode=", "--var=title=Title,with=equals", "--endpoint=" + server.URL + "/v1", "--model=chosen-model", "--temperature=0", "--max-tokens=0", "--top-p=0", "--timeout-seconds=0", "--reasoning-effort=", "--service-tier=priority", `--extra-params={"seed":0,"test_payload":[0,"x"]}`, "--api-key-env=SCRIPTORIUM_CONTROLLED_KEY", "--transient-retries=0", "--run-timeout=5s", "--format=json"} outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 0 || strings.Contains(output+diagnostics, "synthetic-credential-canary") { t.Fatalf("run: %+v %q %q", outcome, output, diagnostics) } var result promptkit.RunResult if err := json.Unmarshal([]byte(output), &result); err != nil { t.Fatal(err) } if string(result.Artifact.Body) != content || result.PromptID != "support/summary" || result.SelectedProfileID != "support/writer" || result.SelectedBackendID != "support/local" || result.SessionID != "caller-session" { t.Fatalf("native result/context: %+v", result) } params := result.EffectiveModelParams if params.Model != "chosen-model" || params.Temperature != 0 || params.MaxTokens != 0 || params.TopP != 0 || params.TimeoutSeconds != 0 || params.ReasoningEffort != "" || params.ServiceTier != "priority" { t.Fatalf("native explicit overrides: %+v", params) } select { case request := <-requests: messages := request["messages"].([]any) message := messages[0].(map[string]any)["content"].(string) for _, want := range []string{"file,transcript=body", "last,literal=2", "Title,with=equals"} { if !strings.Contains(message, want) { t.Fatalf("lost literal/mapping %q: %s", want, message) } } for _, key := range []string{"temperature", "max_tokens", "top_p", "seed"} { if value, exists := request[key]; !exists || value != float64(0) { t.Fatalf("explicit zero %s lost: %+v", key, request) } } if request["model"] != "chosen-model" || request["service_tier"] != "priority" || request["authorization"] != "Bearer synthetic-credential-canary" || request["stale"] != nil || request["reasoning_effort"] != nil { t.Fatalf("provider overrides: %+v", request) } default: t.Fatal("provider was not called") } } func TestRunDeliversRawAndNativeStructuredResults(t *testing.T) { const content = " &\x00\nno-final-newline" var calls atomic.Int64 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": content}}}}) })) defer server.Close() nativePromptFixture(t, server.URL+"/v1") flags := []string{"support/summary", "--config=app.yml", "--llm-profile=support/writer", "--llm-session-id=session", "--llm-input=transcript=text", "--llm-input-text=unlisted=literal", "--llm-var=mode=", "--llm-var=title=Title"} for _, format := range []string{"", "text", "json", "yaml"} { args := append([]string{"run"}, flags...) if format != "" { args = append(args, "--format="+format) } outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 0 { t.Fatalf("%s: %+v %q %q", format, outcome, output, diagnostics) } if format == "" || format == "text" { if output != content || diagnostics != "" { t.Fatalf("raw output decorated: %q %q", output, diagnostics) } } else { var report map[string]any if err := yaml.Unmarshal([]byte(output), &report); err != nil { t.Fatal(err) } if report["prompt_id"] != "support/summary" || report["run_id"] == nil || report["artifact"] == nil || report["result"] != nil || report["schema_version"] != nil { t.Fatalf("non-native result: %+v", report) } } } outcome, output, diagnostics := executeNative(t, append(append([]string{"run"}, flags...), "--verbose")...) if outcome.Status != 0 || output != content || diagnostics == "" { t.Fatalf("verbose streams: %+v %q %q", outcome, output, diagnostics) } if calls.Load() != 5 { t.Fatalf("generation dispatches=%d", calls.Load()) } // The same selected prompt/profile/context used above is valid through the // native preview and requirements paths (Narratio's consistency scenario). discovered := discoverNative(t, "support/summary", "--profile=support/writer", "--session-id=session", "--var=mode=") if discovered.PromptID != "support/summary" || discovered.ProfileID != "support/writer" { t.Fatal(discovered) } } func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing.T) { var fail atomic.Bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if fail.Load() { w.WriteHeader(401) fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`) return } fmt.Fprint(w, `{"choices":[{"message":{"content":"not-json"}}]}`) })) defer server.Close() nativePromptFixture(t, server.URL+"/v1") writeNativeFile(t, "invalid-content.yml", "profiles:\n - id: local\n endpoint: "+server.URL+"/v1\n model: controlled\nprompts:\n - id: json\n default_profile: local\n messages:\n - role: user\n content: Return JSON\n output:\n format: json\n validation_mode: json\n") writeNativeFile(t, "json-app.yml", "promptkit:\n configuration: invalid-content.yml\ntransient_retries: 0\n") args := []string{"run", "json", "--config=json-app.yml"} outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 4 || outcome.Kind != commandkit.OutcomeDomain || outcome.Domain != "validation_failed" || output != "not-json" || diagnostics == "" { t.Fatalf("completed invalid content: %+v %q %q", outcome, output, diagnostics) } outcome, output, diagnostics = executeNative(t, append(args, "--format=json")...) var result promptkit.RunResult if err := json.Unmarshal([]byte(output), &result); err != nil { t.Fatal(err) } if outcome.Status != 4 || result.Validation.Status != promptkit.ValidationFailed || string(result.Artifact.Body) != "not-json" { t.Fatalf("invalid result lost: %+v %+v", outcome, result) } fail.Store(true) outcome, output, diagnostics = executeNative(t, args...) var generation *promptkit.GenerationError if outcome.Status != 1 || output != "" || !errors.Is(outcome.Cause, promptkit.ErrLLMGenerate) || !errors.As(outcome.Cause, &generation) || diagnostics == "" || strings.Contains(diagnostics, "provider-private-canary") { t.Fatalf("operational failure: %+v %q %q", outcome, output, diagnostics) } } func TestRunUsageAndConfigurationFailures(t *testing.T) { t.Chdir(t.TempDir()) for _, args := range [][]string{ {"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"}, {"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"}, {"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"}, {"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"}, } { outcome, _, _ := executeNative(t, args...) if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 { t.Fatalf("%v: %+v", args, outcome) } } writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n") outcome, output, diagnostics := executeNative(t, "run", "one", "--config=invalid.yml") if outcome.Status != 2 || output != "" || !strings.Contains(diagnostics, "validate.error") { t.Fatalf("invalid config: %+v %q %q", outcome, output, diagnostics) } outcome, output, diagnostics = executeNative(t, "run", "one", "--config=absent.yml") if outcome.Status != 1 || output != "" || diagnostics == "" { t.Fatalf("missing config: %+v %q %q", outcome, output, diagnostics) } } func TestTypedRequestFailureMapping(t *testing.T) { _, invalid := configkit.Load(configkit.Bytes("invalid", []byte("[\n")), configkit.Options[map[string]any]{}) for _, test := range []struct { err error status int }{ {fmt.Errorf("wrapped: %w", invalid), 2}, {promptkit.ErrInvalidConfig, 1}, {errors.Join(invalid, context.Canceled), 1}, {errors.Join(invalid, promptkit.ErrOperationTimeout, context.DeadlineExceeded), 1}, {promptkit.ErrCapacityExceeded, 1}, {promptkit.ErrValidation, 1}, {promptkit.ErrInvalidRequest, 1}, {promptkit.ErrPromptNotFound, 1}, {promptkit.ErrLLMGenerate, 1}, } { outcome := requestFailure(test.err, commandkit.Explanation{}) if outcome.Status != test.status || !errors.Is(outcome.Cause, test.err) { t.Fatalf("%v: %+v", test.err, outcome) } } if outcome := completedRunOutcome(nil); outcome.Status != 1 || outcome.Cause == nil { t.Fatalf("nil successful result accepted: %+v", outcome) } // Mapping a duration adds no local positivity policy or second timer. for _, budget := range []time.Duration{-time.Second, 0, time.Second} { request := requestFromRunInput(runInput{OperationTimeout: budget}) if request.OperationTimeout != budget { t.Fatal("operation budget changed", budget) } } } func TestRunDiagnosticQueueKeepsArtifactOnStdout(t *testing.T) { app, err := commandkit.New(commandkit.Identity{Name: "reporting"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome { queueDiagnostics(inv.Reporter(), []commandkit.Diagnostic{commandkit.MessageDiagnostic(commandkit.SeverityInfo, "acquisition-info"), commandkit.MessageDiagnostic(commandkit.SeverityWarning, "acquisition-warning")}) _ = inv.Reporter().Report(commandkit.Report{Value: &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("raw")}}, TextTemplate: artifactTextTemplate}) return commandkit.Success() }})) if err != nil { t.Fatal(err) } var output, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Status != 0 || output.String() != "raw" || !strings.Contains(diagnostics.String(), "acquisition-info") || !strings.Contains(diagnostics.String(), "acquisition-warning") { t.Fatalf("streams: %+v %q %q", outcome, output.String(), diagnostics.String()) } }