From 077290191c550757d993474679dd9226b3ed83e0 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Sat, 21 Sep 2024 23:54:13 -0700 Subject: [PATCH 1/5] add files --- requirements-common.txt | 2 +- use_existing_torch.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 use_existing_torch.py diff --git a/requirements-common.txt b/requirements-common.txt index ad53395307ec..5c617c43829b 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -18,7 +18,7 @@ prometheus_client >= 0.18.0 prometheus-fastapi-instrumentator >= 7.0.0 tiktoken >= 0.6.0 # Required for DBRX tokenizer lm-format-enforcer == 0.10.6 -outlines >= 0.0.43, < 0.1 # Requires torch >= 2.1.0 +outlines >= 0.0.43, < 0.1 typing_extensions >= 4.10 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4 partial-json-parser # used for parsing partial JSON outputs diff --git a/use_existing_torch.py b/use_existing_torch.py new file mode 100644 index 000000000000..a3979b744f1f --- /dev/null +++ b/use_existing_torch.py @@ -0,0 +1,17 @@ +import glob +requires_files = glob.glob('requirements*.txt') +requires_files += ["pyproject.toml"] +for file in requires_files: + print(f">>> cleaning {file}") + with open(file, 'r') as f: + lines = f.readlines() + if "torch" in "".join(lines).lower(): + print("removed:") + with open(file, 'w') as f: + for line in lines: + if 'torch' not in line.lower(): + f.write(line) + else: + print(line.strip()) + print(f"<<< done cleaning {file}") + print() \ No newline at end of file From d1fa1e1bbe2dd669a311cc5ca3f9b81078cf3448 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Sat, 21 Sep 2024 23:58:35 -0700 Subject: [PATCH 2/5] how to use existing pytorch --- docs/source/getting_started/installation.rst | 12 ++++++++++++ use_existing_torch.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index 0322503a89a5..2ee4a1458855 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -72,6 +72,18 @@ You can also build and install vLLM from source: $ cd vllm $ pip install -e . # This may take 5-10 minutes. +.. note:: + + This will uninstall existing PyTorch, and install the version required by vLLM. If you want to use an existing PyTorch installation, please execute ``python use_existing_torch.py`` before running ``pip install -e .``. This command will remove all the PyTorch versions in the requirements files, so that the existing PyTorch installation will be used. For example, a common use case is to build vLLM with PyTorch nightly: + + .. code-block:: console + + $ pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121 + $ git clone https://github.com/vllm-project/vllm.git + $ cd vllm + $ python use_existing_torch.py + $ pip install -e . + .. note:: vLLM can fully run only on Linux, but you can still build it on other systems (for example, macOS). This build is only for development purposes, allowing for imports and a more convenient dev environment. The binaries will not be compiled and not work on non-Linux systems. You can create such a build with the following commands: diff --git a/use_existing_torch.py b/use_existing_torch.py index a3979b744f1f..e11746459908 100644 --- a/use_existing_torch.py +++ b/use_existing_torch.py @@ -1,4 +1,5 @@ import glob + requires_files = glob.glob('requirements*.txt') requires_files += ["pyproject.toml"] for file in requires_files: @@ -14,4 +15,4 @@ else: print(line.strip()) print(f"<<< done cleaning {file}") - print() \ No newline at end of file + print() From 9ba388d6b531e09c1dd63a8d2fbd24c06531ea1c Mon Sep 17 00:00:00 2001 From: youkaichao Date: Sun, 22 Sep 2024 00:17:17 -0700 Subject: [PATCH 3/5] polish --- docs/source/getting_started/installation.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index 2ee4a1458855..6e5adb8ecfa8 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -74,7 +74,7 @@ You can also build and install vLLM from source: .. note:: - This will uninstall existing PyTorch, and install the version required by vLLM. If you want to use an existing PyTorch installation, please execute ``python use_existing_torch.py`` before running ``pip install -e .``. This command will remove all the PyTorch versions in the requirements files, so that the existing PyTorch installation will be used. For example, a common use case is to build vLLM with PyTorch nightly: + This will uninstall existing PyTorch, and install the version required by vLLM. If you want to use an existing PyTorch installation (e.g. build vLLM with PyTorch nightly), there need to be some changes: .. code-block:: console @@ -82,7 +82,14 @@ You can also build and install vLLM from source: $ git clone https://github.com/vllm-project/vllm.git $ cd vllm $ python use_existing_torch.py - $ pip install -e . + $ pip install -r requirements-build.txt + $ pip install -e . --no-build-isolation + + The differences are: + + - ``python use_existing_torch.py``: This script will remove all the PyTorch versions in the requirements files, so that the existing PyTorch installation will be used. + - ``pip install -r requirements-build.txt``: You need to manually install the requirements for building vLLM. + - ``pip install -e . --no-build-isolation``: You need to disable build isolation, so that the build system can use the existing PyTorch installation. .. note:: From 1dae765d501e9a24ee3e41ee41db3e30d87f694b Mon Sep 17 00:00:00 2001 From: youkaichao Date: Sun, 22 Sep 2024 01:20:50 -0700 Subject: [PATCH 4/5] finish doc --- docs/source/getting_started/installation.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index 6e5adb8ecfa8..bf7e55a61dcc 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -74,11 +74,10 @@ You can also build and install vLLM from source: .. note:: - This will uninstall existing PyTorch, and install the version required by vLLM. If you want to use an existing PyTorch installation (e.g. build vLLM with PyTorch nightly), there need to be some changes: + This will uninstall existing PyTorch, and install the version required by vLLM. If you want to use an existing PyTorch installation, there need to be some changes: .. code-block:: console - $ pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121 $ git clone https://github.com/vllm-project/vllm.git $ cd vllm $ python use_existing_torch.py @@ -91,6 +90,11 @@ You can also build and install vLLM from source: - ``pip install -r requirements-build.txt``: You need to manually install the requirements for building vLLM. - ``pip install -e . --no-build-isolation``: You need to disable build isolation, so that the build system can use the existing PyTorch installation. + This is especially useful when the PyTorch dependency cannot be easily installed via pip, e.g.: + + - build vLLM with PyTorch nightly or a custom PyTorch build. + - build vLLM with aarch64 and cuda (GH200), where the PyTorch wheels are not available on PyPI. Currently, only PyTorch nightly has wheels for aarch64 with CUDA. You can run ``pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121`` to install PyTorch nightly, and then build vLLM on top of it. + .. note:: vLLM can fully run only on Linux, but you can still build it on other systems (for example, macOS). This build is only for development purposes, allowing for imports and a more convenient dev environment. The binaries will not be compiled and not work on non-Linux systems. You can create such a build with the following commands: From a9309a7d70f200c95b751b13ec33bc8487b10e19 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Sun, 22 Sep 2024 09:12:25 -0700 Subject: [PATCH 5/5] change cuda version --- docs/source/getting_started/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index bf7e55a61dcc..afae6e655602 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -93,7 +93,7 @@ You can also build and install vLLM from source: This is especially useful when the PyTorch dependency cannot be easily installed via pip, e.g.: - build vLLM with PyTorch nightly or a custom PyTorch build. - - build vLLM with aarch64 and cuda (GH200), where the PyTorch wheels are not available on PyPI. Currently, only PyTorch nightly has wheels for aarch64 with CUDA. You can run ``pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121`` to install PyTorch nightly, and then build vLLM on top of it. + - build vLLM with aarch64 and cuda (GH200), where the PyTorch wheels are not available on PyPI. Currently, only PyTorch nightly has wheels for aarch64 with CUDA. You can run ``pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124`` to install PyTorch nightly, and then build vLLM on top of it. .. note::