✨ vllm support for 0.11.1 release

.github/actions/free-up-disk-space/action.yml

@@ -0,0 +1,46 @@
+name: "Free up disk space"
+description: "Removes non-essential tools, libraries and cached files from GitHub action runner node"
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Remove non-essential tools and libraries"
+      shell: bash
+      run: |
+        # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+        echo "Disk usage before cleanup:"
+        df -h
+        echo "Removing non-essential tools and libraries ..."
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /usr/local/.ghcup
+        sudo rm -rf /usr/share/dotnet
+        # sudo rm -rf /usr/local/share/boost
+        echo "Deleting libraries for Android (12G), CodeQL (5.3G), PowerShell (1.3G), Swift (1.7G) ..."
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf "${AGENT_TOOLSDIRECTORY}/CodeQL"
+        sudo rm -rf /usr/local/share/powershell
+        sudo rm -rf /usr/share/swift
+        # ref: https://github.com/jlumbroso/free-disk-space/blob/main/action.yml
+        echo "Deleting some larger apt packages:"
+        sudo apt-get remove -y '^aspnetcore-.*' || echo "::warning::The command [sudo apt-get remove -y '^aspnetcore-.*'] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^dotnet-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^llvm-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^llvm-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y 'php.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y 'php.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mongodb-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mysql-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "::warning::The command [sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y google-cloud-sdk --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-sdk --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y google-cloud-cli --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-cli --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed to complete successfully. Proceeding..."
+        sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed to complete successfully. Proceeding..."
+        echo "Disk usage after cleanup:"
+        df -h
+
+    - name: "Prune docker images"
+      shell: bash
+      run: |
+        echo "Pruning docker images ..."
+        docker image prune -a -f
+        docker system df
+        echo "Disk usage after pruning docker images:"
+        df -h

.github/workflows/build_docker.yml

@@ -43,6 +43,9 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: "Free up disk space"
+      uses: ./.github/actions/free-up-disk-space
+
     - name: "Set up QEMU"
       uses: docker/setup-qemu-action@v3
 

.github/workflows/test.yml

@@ -79,6 +79,7 @@ jobs:
             markers: "precompilation and not quantized"
             flags: "--timeout=300"
         include:
+          # Lower bound support
           - vllm_version:
               name: "vLLM:lowest"
               repo: "git+https://github.com/vllm-project/vllm --tag v0.10.2"
@@ -90,6 +91,19 @@ jobs:
               hf_model_2_rev: "cf74d8acd4f198de950bf004b262e6accfed5d2c"
             os: "ubuntu-latest"
             python_version: "3.12"
+          # Intermediate versions of vllm to check basic support for as well
+          - vllm_version:
+              name: "vLLM:0.11.0"
+              repo: "git+https://github.com/vllm-project/vllm --tag v0.11.0"
+            test_suite:
+              name: "backward compat"
+              markers: "compat or (cpu and basic)"
+              flags: "--timeout=300"
+              hf_model_2: "sentence-transformers/all-roberta-large-v1"
+              hf_model_2_rev: "cf74d8acd4f198de950bf004b262e6accfed5d2c"
+            os: "ubuntu-latest"
+            python_version: "3.12"
+
         # Exclude vLLM:main if PR does NOT have "ready" label AND auto-merge is not enabled
         exclude: >-
           ${{

pyproject.toml

@@ -11,9 +11,9 @@ description = "vLLM plugin for Spyre hardware support"
 readme = "README.md"
 license = {text = "Apache 2"}
 dependencies = [
-    "fms-model-optimizer[fp8]>=0.6.0",
+    "fms-model-optimizer[fp8]>=0.8.0",
     "ibm-fms>=1.5.0,<2.0",
-    "vllm>=0.10.2,<=0.11.0",
+    "vllm>=0.10.2,<=0.11.1",
     "pytest-mock>=3.15.0",
 ]
 requires-python = ">=3.11"
@@ -50,6 +50,7 @@ override-dependencies = [
     "torchaudio; sys_platform == 'never'",
     "torchvision; sys_platform == 'never'",
     "triton; sys_platform == 'never'",
+    "intel-extension-for-pytorch; sys_platform == 'never'",
 
     # Skip packages on s390x and ppc64le, expected to be pre-installed
     "vllm ; platform_machine not in 's390x, ppc64le'",
@@ -62,6 +63,9 @@ environments = [
     "python_version > '3.9'"
 ]
 
+[tool.uv.sources]
+vllm = { git = "https://github.com/vllm-project/vllm", rev = "v0.11.1" }
+
 [tool.ruff]
 # Allow lines to be as long as 80.
 line-length = 80

tests/e2e/test_chunked_prefill_tkv_steps.py

@@ -20,6 +20,7 @@
 
 from vllm_spyre.platform import SpyrePlatform
 from vllm_spyre.v1.worker.spyre_model_runner import SpyreModelRunner
+from vllm_spyre.v1.worker.spyre_worker import _get_extra_args
 
 
 ########## Assuming that we have:
@@ -107,6 +108,7 @@ def make_scheduler_output(
     if finished_req_ids is None:
         finished_req_ids = set()
 
+    extra_args = _get_extra_args()
     return SchedulerOutput(scheduled_new_reqs=scheduled_new_reqs,
                            scheduled_cached_reqs=scheduled_cached_reqs,
                            num_scheduled_tokens=num_scheduled_tokens,
@@ -115,10 +117,8 @@ def make_scheduler_output(
                            scheduled_encoder_inputs={},
                            num_common_prefix_blocks=[],
                            finished_req_ids=finished_req_ids,
-                           free_encoder_mm_hashes=[],
-                           structured_output_request_ids={},
-                           grammar_bitmask=None,
-                           kv_connector_metadata=None)
+                           kv_connector_metadata=None,
+                           **extra_args)
 
 
 def make_new_request_data(req_id, prompt_len):

tests/e2e/test_spyre_cb.py

@@ -37,7 +37,10 @@ def test_cb_max_tokens(model: ModelInfo, backend: str, max_model_len: int,
                                           ignore_eos=True,
                                           logprobs=0)
 
-    with pytest.raises(ValueError, match="max model context length"):
+    # The text of the error raised by vllm changed from 0.11.0 to 0.11.1
+    with pytest.raises(
+            ValueError,
+            match="(max model context length|maximum model length)"):
         generate_spyre_vllm_output(
             model=model,
             prompts=overflow_prompt,

tests/llm_cache.py

@@ -234,7 +234,7 @@ def get_engine(
 
         # Set scheduler configs for max_model_len and max_num_seqs to the
         # original values. They were changed for more robust compilation only.
-        engine_core.scheduler.scheduler_config.max_model_len = max_model_len
+        engine_core.scheduler.model_config.max_model_len = max_model_len
         engine_core.scheduler.scheduler_config.max_num_seqs = max_num_seqs
 
         if available_blocks is not None:

tests/spyre_util.py

@@ -14,7 +14,15 @@
 from transformers import AutoTokenizer
 from vllm import SamplingParams
 from vllm.entrypoints.openai.cli_args import make_arg_parser
-from vllm.utils import FlexibleArgumentParser, get_open_port
+
+try:
+    # old
+    from vllm.utils import FlexibleArgumentParser, get_open_port
+except ImportError:
+    # new
+    from vllm.utils.argparse_utils import FlexibleArgumentParser
+    from vllm.utils.network_utils import get_open_port
+
 from vllm.v1.request import Request
 
 EmbeddingWarmupShapes = list[tuple[int, int]]

tests/utils/test_upstream_compatibility.py

@@ -1,7 +1,7 @@
 import os
 
 import pytest
-from vllm.v1.core.sched.output import NewRequestData
+from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
 from vllm.v1.kv_cache_interface import FullAttentionSpec
 
 from vllm_spyre.compat_utils import dataclass_fields
@@ -33,3 +33,40 @@ def test_use_mla():
     if VLLM_VERSION == "vLLM:lowest":
         # Can remove backwards compatibility for use_mla
         assert "use_mla" in dataclass_fields(FullAttentionSpec)
+
+
+def test_pin_memory_available():
+    if VLLM_VERSION == "vLLM:lowest":
+        try:
+            from vllm.utils import is_pin_memory_available  # # noqa
+            from vllm.utils import make_tensor_with_pad  # # noqa
+        except ImportError as e:
+            raise AssertionError("remove backwards compatibility imports for "
+                                 "is_pin_memory_available and "
+                                 "make_tensor_with_pad") from e
+
+
+def test_multi_modal_cache_stats():
+    if VLLM_VERSION == "vLLM:lowest":
+        # If this import succeeds then remove the backwards compatibility type
+        # def for MultiModalCacheStats
+        with pytest.raises(ImportError):
+            from vllm.v1.metrics.stats import MultiModalCacheStats  # # noqa
+
+
+def test_v0_worker_base():
+    if VLLM_VERSION == "vLLM:lowest":
+        try:
+            from vllm.worker.worker_base import WorkerBase  # # noqa
+        except ImportError as e:
+            raise AssertionError(
+                "remove the backwards compatibility code from "
+                "the SpyreWorker initializer") from e
+
+
+def test_structured_output_request_ids():
+    if VLLM_VERSION == "vLLM:lowest":
+        # Can remove "structured_output_request_ids" and "grammar_bitmask"
+        # from backwards compat
+        assert 'structured_output_request_ids' in dataclass_fields(
+            SchedulerOutput)

tests/v1/worker/test_spyre_input_batch.py

@@ -6,7 +6,13 @@
 import pytest
 import torch
 from vllm.sampling_params import SamplingParams
-from vllm.utils import is_pin_memory_available, make_tensor_with_pad
+
+try:
+    from vllm.utils import is_pin_memory_available, make_tensor_with_pad
+except ImportError:
+    from vllm.utils.platform_utils import is_pin_memory_available
+    from vllm.utils.torch_utils import make_tensor_with_pad
+
 from vllm.v1.sample.logits_processor import LogitsProcessors
 from vllm.v1.sample.metadata import SamplingMetadata