⬆️ bump vllm upper bound to support 0.10.2

.github/workflows/test.yml

@@ -69,7 +69,7 @@ jobs:
         include:
           - vllm_version:
               name: "vLLM:lowest"
-              repo: "git+https://github.com/vllm-project/vllm --tag v0.10.0"
+              repo: "git+https://github.com/vllm-project/vllm --tag v0.10.1.1"
             test_suite:
               name: "backward compat"
               markers: "compat or (cpu and basic)"

pyproject.toml

@@ -13,7 +13,7 @@ license = {text = "Apache 2"}
 dependencies = [
     "fms-model-optimizer[fp8]>=0.6.0",
     "ibm-fms>=1.2.1",
-    "vllm>=0.10.0,<=0.10.1.1",
+    "vllm>=0.10.1.1,<=0.10.2",
 ]
 requires-python = ">=3.11"
 dynamic = ["version"]
@@ -163,7 +163,7 @@ dev = [
     "pytest-timeout==2.3.1",
     "requests==2.32.3",
     "sentence-transformers==3.4.1",
-    "aiu-fms-testing-utils>=0.2.1",
+    "aiu-fms-testing-utils>=0.2.3",
 ]
 lint = [
     "clang-format==18.1.5",

tests/e2e/test_spyre_cb.py

@@ -64,8 +64,7 @@ def test_api_cb_rejects_oversized_request(
     overflow_prompt = " ".join(["hi"] * max_model_len)
     max_tokens = 10
 
-    with pytest.raises(BadRequestError,
-                       match="This model's maximum context length is"):
+    with pytest.raises(BadRequestError, match="maximum context length is"):
         client.completions.create(
             model=model.name,
             prompt=overflow_prompt,

tests/spyre_util.py

@@ -372,14 +372,12 @@ def create_random_request(
             cache_salt=None,
             **kwargs,
         )
-    kwargs = {
-        "multi_modal_kwargs" if inputs_renamed else "multi_modal_inputs": None
-    }
+    kwargs = {}
+    if inputs_renamed:
+        kwargs = {"multi_modal_kwargs"}
     return Request(
         request_id=str(request_id),
         prompt_token_ids=prompt_token_ids,
-        multi_modal_hashes=None,
-        multi_modal_placeholders=None,
         sampling_params=sampling_params,
         eos_token_id=None,
         arrival_time=0,

vllm_spyre/platform.py

@@ -43,6 +43,13 @@
 ]
 
 
+# Add comment
+class _StreamPlaceholder:
+
+    def __init__(self):
+        self.synchronize = lambda: None
+
+
 class classproperty:
 
     def __init__(self, func):
@@ -80,6 +87,9 @@ class SpyrePlatform(Platform):
     # See vllm batched_count_greater_than method
     # simple_compile_backend: str = "eager"
 
+    # ADD COMMENT
+    current_stream = lambda _: _StreamPlaceholder()
+
     @classproperty
     def device_type(cls):
         # TODO: temporary hack while BertModels
@@ -106,6 +116,12 @@ def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
 
     @classmethod
     def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
+
+        # in case vllm passes a default vllm_config to us
+        # add some more comments as to why this needed
+        if vllm_config.model_config is None:
+            return
+
         cls._config = vllm_config
         parallel_config = vllm_config.parallel_config
         scheduler_config = vllm_config.scheduler_config

vllm_spyre/v1/worker/spyre_model_runner.py

@@ -12,7 +12,7 @@
 from torch import nn
 from transformers import (AutoModel, AutoModelForSequenceClassification,
                           AutoTokenizer)
-from vllm.config import DeviceConfig, VllmConfig
+from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
 from vllm.forward_context import set_forward_context
 from vllm.logger import init_logger
 from vllm.model_executor.layers.pooler import (ClassifierPooler, Pooler,
@@ -558,7 +558,6 @@ def execute_model(
             req_ids=list(req_id_to_index.keys()),
             req_id_to_index=req_id_to_index,
             sampled_token_ids=output.sampled_token_ids.tolist(),
-            spec_token_ids=None,
             logprobs=(output.logprobs_tensors.tolists()
                       if output.logprobs_tensors else None),
             prompt_logprobs_dict=prompt_logprobs_dicts,
@@ -1448,15 +1447,17 @@ def load_model(self, prompt_lens: Iterable[int],
             extra_args['default_pooling_type'] = PoolingType.CLS
 
         if task == "embed":
-            self.pooler = Pooler.for_embed(pooler_config=pooler_config,
-                                           **extra_args)
+            with set_current_vllm_config(self.vllm_config):
+                self.pooler = Pooler.for_embed(pooler_config=pooler_config,
+                                               **extra_args)
         elif task == "classify":
-            self.pooler = ClassifierPooler(
-                pooling=self._pooler,
-                classifier=ClassifierAdapter(self.classifier),
-                act_fn=ClassifierPooler.act_fn_for_cross_encoder(
-                    self.model_config),
-            )
+            with set_current_vllm_config(self.vllm_config):
+                self.pooler = ClassifierPooler(
+                    pooling=self._pooler,
+                    classifier=ClassifierAdapter(self.classifier),
+                    act_fn=ClassifierPooler.act_fn_for_cross_encoder(
+                        self.model_config),
+                )
 
     @property
     def vocab_size(self) -> int:
@@ -1630,6 +1631,10 @@ def execute_model(
         logger.debug("t_batch: %.2fms", (t1 * 1000))
 
         pooling_metadata = self.input_batch.make_pooling_metadata()
+        ## No partial prefill, hence
+        pooling_metadata.build_pooling_cursor(
+            num_scheduled_tokens=pooling_metadata.prompt_lens,
+            device=self.device)
 
         # prepare unpadded output for the pooler
         hidden_state_list: list[torch.Tensor] = []
@@ -1638,8 +1643,9 @@ def execute_model(
             # we're left padding
             hidden_state_list.append(hidden_state[-prompt_len:])
 
-        raw_pooler_output = self.pooler(hidden_states=hidden_state_list,
-                                        pooling_metadata=pooling_metadata)
+        raw_pooler_output = self.pooler(
+            hidden_states=torch.cat(hidden_state_list),
+            pooling_metadata=pooling_metadata)
 
         pooler_output: list[Optional[torch.Tensor]] = []
 
@@ -1650,7 +1656,6 @@ def execute_model(
             req_ids=self.input_batch.requests_ids,
             req_id_to_index=self.input_batch.req_id_to_index,
             sampled_token_ids=[],
-            spec_token_ids=None,
             logprobs=None,
             prompt_logprobs_dict={},
             pooler_output=pooler_output,

vllm_spyre/v1/worker/spyre_worker.py

@@ -476,7 +476,7 @@ def _warmup_spyre_dynamic_size(self, special_token_ids):
             scheduled_encoder_inputs={},
             num_common_prefix_blocks=0,
             finished_req_ids=set(),
-            free_encoder_input_ids=[],
+            free_encoder_mm_hashes=[],
             structured_output_request_ids={},
             grammar_bitmask=None,
         )
@@ -508,7 +508,7 @@ def _cleanup_model_runner(self, request) -> None:
             num_common_prefix_blocks=0,
             # The requests to be removed
             finished_req_ids=set([r.req_id for r in request]),
-            free_encoder_input_ids=[],
+            free_encoder_mm_hashes=[],
             structured_output_request_ids={},
             grammar_bitmask=None,
         )
@@ -590,7 +590,7 @@ def _warmup_spyre_fixed_size(self, prompt_len, num_decode_tokens,
             scheduled_encoder_inputs={},
             num_common_prefix_blocks=0,
             finished_req_ids=set(),
-            free_encoder_input_ids=[],
+            free_encoder_mm_hashes=[],
             structured_output_request_ids={},
             grammar_bitmask=None,
         )
@@ -655,7 +655,7 @@ def _dynamic_warmup(
                 scheduled_encoder_inputs={},
                 num_common_prefix_blocks=0,
                 finished_req_ids=set(),
-                free_encoder_input_ids=[],
+                free_encoder_mm_hashes=[],
                 structured_output_request_ids={},
                 grammar_bitmask=None,
             )
@@ -692,7 +692,7 @@ def _dynamic_warmup(
             scheduled_encoder_inputs={},
             num_common_prefix_blocks=0,
             finished_req_ids=set(),
-            free_encoder_input_ids=[],
+            free_encoder_mm_hashes=[],
             structured_output_request_ids={},
             grammar_bitmask=None,
         )