From aedcd842bc6dcc7c82afbf02605bd26f497b59a9 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Thu, 6 Mar 2025 09:24:47 +0000 Subject: [PATCH 01/10] update torch 2.6 Signed-off-by: jiang1.li --- .buildkite/run-cpu-test.sh | 1 + Dockerfile.cpu | 2 +- requirements/build.txt | 2 +- requirements/cpu.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/run-cpu-test.sh b/.buildkite/run-cpu-test.sh index f6dad818ddc0..e1fd2a7d1828 100644 --- a/.buildkite/run-cpu-test.sh +++ b/.buildkite/run-cpu-test.sh @@ -36,6 +36,7 @@ function cpu_tests() { docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c " set -e pip install -r vllm/requirements/test.txt + pip install -r vllm/requirements/cpu.txt pytest -v -s tests/models/decoder_only/language -m cpu_model pytest -v -s tests/models/embedding/language -m cpu_model pytest -v -s tests/models/encoder_decoder/language -m cpu_model diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 08a4e188f4c1..a10090529d8a 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -22,7 +22,7 @@ ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/li RUN echo 'ulimit -c 0' >> ~/.bashrc -RUN pip install intel_extension_for_pytorch==2.5.0 +RUN pip install intel_extension_for_pytorch==2.6.0 WORKDIR /workspace diff --git a/requirements/build.txt b/requirements/build.txt index fec01caaf25e..364a16d80b71 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -4,6 +4,6 @@ ninja packaging setuptools>=61 setuptools-scm>=8 -torch==2.5.1 +torch==2.6.0 wheel jinja2 diff --git a/requirements/cpu.txt b/requirements/cpu.txt index ba059d3ff72e..ff4b4831175b 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -2,7 +2,7 @@ -r common.txt # Dependencies for CPUs -torch==2.5.1+cpu; platform_machine != "ppc64le" and platform_machine != "aarch64" and platform_system != "Darwin" and platform_machine != "s390x" +torch==2.6.0+cpu; platform_machine != "ppc64le" and platform_machine != "aarch64" and platform_system != "Darwin" and platform_machine != "s390x" torch==2.5.1; platform_machine == "ppc64le" or platform_machine == "aarch64" or platform_system == "Darwin" torch==2.7.0.dev20250304; platform_machine == "s390x" From ea01282139a84160db81a0adfa99d36fe4031a01 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Thu, 6 Mar 2025 09:36:59 +0000 Subject: [PATCH 02/10] fix cuda error msg Signed-off-by: jiang1.li --- vllm/executor/multiproc_worker_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vllm/executor/multiproc_worker_utils.py b/vllm/executor/multiproc_worker_utils.py index 68a83bb610a4..74237f9eb451 100644 --- a/vllm/executor/multiproc_worker_utils.py +++ b/vllm/executor/multiproc_worker_utils.py @@ -254,10 +254,11 @@ def _run_worker_process( # online (in situ) tuning is enabled. # Offline tuning API (record_untuned_is_enabled()) only # available in PyTorch 2.6 or later. - import torch.cuda.tunable as tunable - if (tunable.is_enabled() and tunable.tuning_is_enabled() - and not tunable.record_untuned_is_enabled()): - tunable.write_file() + if torch.cuda.is_available(): + import torch.cuda.tunable as tunable + if (tunable.is_enabled() and tunable.tuning_is_enabled() + and not tunable.record_untuned_is_enabled()): + tunable.write_file() logger.info("Worker exiting") From 4dbed091498cd0751539e7e95c2ec0678c68c064 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Fri, 7 Mar 2025 13:15:58 +0000 Subject: [PATCH 03/10] update moe support Signed-off-by: jiang1.li --- vllm/attention/ops/ipex_attn.py | 2 +- vllm/model_executor/layers/fused_moe/layer.py | 6 +++++- vllm/platforms/cpu.py | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/vllm/attention/ops/ipex_attn.py b/vllm/attention/ops/ipex_attn.py index 598ceea130d9..6d96f58320c8 100644 --- a/vllm/attention/ops/ipex_attn.py +++ b/vllm/attention/ops/ipex_attn.py @@ -17,7 +17,7 @@ class _PagedAttention: @staticmethod def get_supported_head_sizes() -> List[int]: - return [32, 64, 80, 96, 112, 128, 256] + return [32, 64, 80, 96, 112, 128, 192, 256] @staticmethod def get_kv_cache_shape( diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 51c4df9d4a5e..2c5fa509c595 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -193,10 +193,11 @@ def forward_cpu( global_num_experts: int = -1, expert_map: Optional[torch.Tensor] = None, custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, activation: str = "silu", **kwargs, ): - assert custom_routing_function is None assert activation == "silu", f"{activation} is not supported." return layer.ipex_fusion( x, @@ -206,6 +207,9 @@ def forward_cpu( renormalize, topk_group, num_expert_group, + custom_routing_function, + scoring_func, + e_score_correction_bias, ) def forward_tpu( diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index ab8982a3a6e1..140335dfb64a 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -121,6 +121,9 @@ def check_and_update_config(cls, vllm_config: VllmConfig) -> None: # Disable torch async compiling which won't work with daemonic processes os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" + # MLA attention is not supported + os.environ["VLLM_MLA_DISABLE"] = "1" + # Intel OpenMP setting ld_prealod_str = os.getenv("LD_PRELOAD", "") if "libiomp5.so" in ld_prealod_str: From ce8e1a8f57612319c24d55ca4599058b9bd06fb9 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Fri, 7 Mar 2025 13:52:28 +0000 Subject: [PATCH 04/10] fix failed lora test Signed-off-by: jiang1.li --- tests/lora/test_qwen2vl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lora/test_qwen2vl.py b/tests/lora/test_qwen2vl.py index 90735d55be71..7bd3e3d0fe27 100644 --- a/tests/lora/test_qwen2vl.py +++ b/tests/lora/test_qwen2vl.py @@ -12,7 +12,7 @@ from vllm.platforms import current_platform -@pytest.fixture(autouse=True) +@pytest.fixture(autouse=not current_platform.is_cpu()) def v1(run_with_both_engines_lora): # Simple autouse wrapper to run both engines for each test # This can be promoted up to conftest.py to run for every From fb8dd8e0bbb2b691ba9d1e0df41aeedd3ee859dc Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Fri, 7 Mar 2025 14:27:21 +0000 Subject: [PATCH 05/10] fix ci Signed-off-by: jiang1.li --- .buildkite/run-cpu-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/run-cpu-test.sh b/.buildkite/run-cpu-test.sh index e1fd2a7d1828..736515b9be79 100644 --- a/.buildkite/run-cpu-test.sh +++ b/.buildkite/run-cpu-test.sh @@ -26,6 +26,7 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg function cpu_tests() { set -e export NUMA_NODE=$2 + export BUILDKITE_BUILD_NUMBER=$3 # offline inference docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" bash -c " @@ -86,4 +87,4 @@ function cpu_tests() { # All of CPU tests are expected to be finished less than 40 mins. export -f cpu_tests -timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE" +timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE $BUILDKITE_BUILD_NUMBER" From b50e63062bdbf810b6eafe0aa7246eb717fa8e00 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Tue, 11 Mar 2025 10:17:52 +0000 Subject: [PATCH 06/10] update oneDNN Signed-off-by: jiang1.li --- cmake/cpu_extension.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index ca2ffb1bc3c8..345b75d62233 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -149,7 +149,7 @@ if (AVX512_FOUND AND NOT AVX512_DISABLED) FetchContent_Declare( oneDNN GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git - GIT_TAG v3.6 + GIT_TAG v3.7.1 GIT_PROGRESS TRUE GIT_SHALLOW TRUE ) From dbb318ccdaaf38bc49144482d4bac82bce68e630 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Tue, 11 Mar 2025 10:34:22 +0000 Subject: [PATCH 07/10] revert change in build requirements Signed-off-by: jiang1.li --- requirements/build.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/build.txt b/requirements/build.txt index 364a16d80b71..fec01caaf25e 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -4,6 +4,6 @@ ninja packaging setuptools>=61 setuptools-scm>=8 -torch==2.6.0 +torch==2.5.1 wheel jinja2 From 105ad93e4caa250e2ef11f0110c121a51c8cacea Mon Sep 17 00:00:00 2001 From: "Li, Jiang" Date: Tue, 11 Mar 2025 19:37:12 +0800 Subject: [PATCH 08/10] Update requirements/cpu.txt Co-authored-by: Isotr0py <2037008807@qq.com> Signed-off-by: jiang1.li --- requirements/cpu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/cpu.txt b/requirements/cpu.txt index ff4b4831175b..0242d070a2cc 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -2,7 +2,7 @@ -r common.txt # Dependencies for CPUs -torch==2.6.0+cpu; platform_machine != "ppc64le" and platform_machine != "aarch64" and platform_system != "Darwin" and platform_machine != "s390x" +torch==2.6.0+cpu; platform_machine == "x86_64" and platform_system != "Darwin" torch==2.5.1; platform_machine == "ppc64le" or platform_machine == "aarch64" or platform_system == "Darwin" torch==2.7.0.dev20250304; platform_machine == "s390x" From 77fbde69932bde1d43ecf3a2e8d5495b7fbfee17 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Tue, 11 Mar 2025 11:38:52 +0000 Subject: [PATCH 09/10] refactor Signed-off-by: jiang1.li --- requirements/cpu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/cpu.txt b/requirements/cpu.txt index 0242d070a2cc..b4e6abb6e3d6 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -2,7 +2,7 @@ -r common.txt # Dependencies for CPUs -torch==2.6.0+cpu; platform_machine == "x86_64" and platform_system != "Darwin" +torch==2.6.0+cpu; platform_machine == "x86_64" torch==2.5.1; platform_machine == "ppc64le" or platform_machine == "aarch64" or platform_system == "Darwin" torch==2.7.0.dev20250304; platform_machine == "s390x" From dedbc1c55c05f15ab793a979cfd59d1f4026b929 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Wed, 12 Mar 2025 03:29:24 +0000 Subject: [PATCH 10/10] fix Signed-off-by: jiang1.li --- .buildkite/run-cpu-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/run-cpu-test.sh b/.buildkite/run-cpu-test.sh index 736515b9be79..e45e184852f2 100644 --- a/.buildkite/run-cpu-test.sh +++ b/.buildkite/run-cpu-test.sh @@ -19,9 +19,9 @@ remove_docker_container # Run the image, setting --shm-size=4g for tensor parallel. docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \ - --cpuset-mems="$NUMA_NODE" --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER" + --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER" docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \ - --cpuset-mems="$NUMA_NODE" --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 + --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2 function cpu_tests() { set -e