From 0d9d612aa7b7ebfe3695591933a94cb801d6ac69 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Thu, 26 Sep 2024 23:09:09 +0000
Subject: [PATCH 01/31] Remove unnecessary warning for missing
 `dummy_encoder_data_factory`

---
 vllm/inputs/registry.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/vllm/inputs/registry.py b/vllm/inputs/registry.py
index 159d958ebf67..eed86ea02713 100644
--- a/vllm/inputs/registry.py
+++ b/vllm/inputs/registry.py
@@ -185,16 +185,8 @@ def wrapper(model_cls: N) -> N:
         return wrapper
 
     def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
-        if model_cls in self._dummy_encoder_factories_by_model_type:
-            dummy_factory = self._dummy_encoder_factories_by_model_type[
-                model_cls]
-        else:
-            logger.warning(
-                "No dummy encoder data factory registered to %s. "
-                "Using the dummy data factory for the model instead.",
-                model_cls)
-            dummy_factory = self._get_dummy_data_factory(model_cls)
-        return dummy_factory
+        return self._dummy_encoder_factories_by_model_type \
+            .get(model_cls, self._default_dummy_data_factory)
 
     def dummy_data_for_profiling(
         self,

From a93a4e111e81f3b2f3fa2bb2c15923ed901f79d5 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Thu, 26 Sep 2024 23:09:16 +0000
Subject: [PATCH 02/31] Actually test mllama

---
 .buildkite/test-pipeline.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index b4226a3ca574..48cfebb264b3 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -330,6 +330,7 @@ steps:
   commands:
     - pytest -v -s models/embedding/language
     - pytest -v -s models/encoder_decoder/language
+    - pytest -v -s models/encoder_decoder/vision_language
 
 #####  1 GPU test  #####
 #####  multi gpus test  #####

From 3241f706e76c23970ce4860dc6e286fad5d94246 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 00:50:52 +0000
Subject: [PATCH 03/31] Use smaller model for llava_onevision and fix chat
 template

---
 .../vision_language/test_llava_onevision.py   | 22 ++++++-------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index 978631feacb8..6852244574a7 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -16,11 +16,10 @@
 # Video test
 HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
     "sample_demo_1":
-    "<|im_start|>user <video>\nwhy is this video funny? \
-    <|im_end|><|im_start|>assistant\n"
+    "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n"  # noqa: E501
 })
 
-models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
+models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
 
 
 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -254,9 +253,8 @@ def run_image_test(
     # max_model_len should be greater than image_feature_size
     with vllm_runner(model,
                      dtype=dtype,
-                     max_num_seqs=1,
                      max_model_len=16384,
-                     gpu_memory_utilization=0.98,
+                     max_num_seqs=2,
                      tensor_parallel_size=tensor_parallel_size,
                      distributed_executor_backend=distributed_executor_backend,
                      enforce_eager=True,
@@ -302,8 +300,6 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
-# FIXME: Swap to a smaller model for this architecture
-@pytest.mark.skip(reason="Model OOMing on CI")
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])
@@ -316,14 +312,10 @@ def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
 
     inputs = [(
         [
-            "<|im_start|>user <image><image>\nDescribe 2 images. \
-                <|im_end|><|im_start|>assistant\n",
-            "<|im_start|>user <image><image>\nDescribe 2 images. \
-                <|im_end|><|im_start|>assistant\n",
-            "<|im_start|>user <image><image><image><image>\nDescribe 4 images. \
-                <|im_end|><|im_start|>assistant\n",
-            "<|im_start|>user <image>\nWhat is the season? \
-                <|im_end|><|im_start|>assistant\n",
+            "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
+            "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
+            "<|im_start|>user\n<image><image><image><image>\nDescribe 4 images.<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
+            "<|im_start|>user\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
         ],
         [
             [stop_sign, cherry_blossom],

From f7e3f89c55689f091ef69b0dc7bbec0dfd7ddccf Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 01:06:11 +0000
Subject: [PATCH 04/31] Fix mllama test

---
 .../models/encoder_decoder/vision_language/test_mllama.py  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/models/encoder_decoder/vision_language/test_mllama.py b/tests/models/encoder_decoder/vision_language/test_mllama.py
index cda0926d0baf..daec00720367 100644
--- a/tests/models/encoder_decoder/vision_language/test_mllama.py
+++ b/tests/models/encoder_decoder/vision_language/test_mllama.py
@@ -47,8 +47,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
         if token_id != image_token_id or output_ids[idx - 1] != image_token_id
     ]
 
-    assert output_str[0] == " "
-    hf_output_str = output_str[1:]
+    hf_output_str = output_str
     if hf_output_ids[-1] == eos_token_id:
         hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
 
@@ -167,8 +166,8 @@ def _run_test(
     # max_model_len should be greater than image_feature_size
     with vllm_runner(model,
                      dtype=dtype,
-                     max_num_seqs=16,
                      max_model_len=4096,
+                     max_num_seqs=2,
                      tensor_parallel_size=tensor_parallel_size,
                      distributed_executor_backend=distributed_executor_backend,
                      enforce_eager=True,
@@ -185,7 +184,6 @@ def _run_test(
     def process(hf_inputs: BatchEncoding):
         return hf_inputs
 
-    from transformers import AutoConfig
     from transformers.models.mllama import MllamaConfig as MllamaConfigHf
 
     # use transformer's MllamaConfig for hf_runner
@@ -257,6 +255,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
 
 
 @multi_gpu_test(num_gpus=2)
+@pytest.mark.skip("ps.get_tp_group().all_gather() used in vLLM but not in HF")
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize(
     "sizes",

From 0efde87074ceed86c2bdb843cbb5e7834900d76c Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 01:06:37 +0000
Subject: [PATCH 05/31] Fix ragged batch shape not being tested because
 `max_num_seqs=1`

---
 tests/models/decoder_only/vision_language/test_fuyu.py     | 2 +-
 tests/models/decoder_only/vision_language/test_minicpmv.py | 2 +-
 tests/models/decoder_only/vision_language/test_phi3v.py    | 2 +-
 tests/models/decoder_only/vision_language/test_qwen.py     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/models/decoder_only/vision_language/test_fuyu.py b/tests/models/decoder_only/vision_language/test_fuyu.py
index 94b8431424db..3ec36cb003cd 100644
--- a/tests/models/decoder_only/vision_language/test_fuyu.py
+++ b/tests/models/decoder_only/vision_language/test_fuyu.py
@@ -66,7 +66,7 @@ def run_test(
     # max_model_len should be greater than image_feature_size
     with vllm_runner(model,
                      max_model_len=2560,
-                     max_num_seqs=1,
+                     max_num_seqs=2,
                      dtype=dtype,
                      tensor_parallel_size=tensor_parallel_size,
                      distributed_executor_backend=distributed_executor_backend,
diff --git a/tests/models/decoder_only/vision_language/test_minicpmv.py b/tests/models/decoder_only/vision_language/test_minicpmv.py
index 7bf5d75f400f..1d4e75205227 100644
--- a/tests/models/decoder_only/vision_language/test_minicpmv.py
+++ b/tests/models/decoder_only/vision_language/test_minicpmv.py
@@ -79,7 +79,7 @@ def run_test(
     # max_model_len should be greater than image_feature_size
     with vllm_runner(model,
                      max_model_len=4096,
-                     max_num_seqs=1,
+                     max_num_seqs=2,
                      dtype=dtype,
                      limit_mm_per_prompt={"image": mm_limit},
                      tensor_parallel_size=tensor_parallel_size,
diff --git a/tests/models/decoder_only/vision_language/test_phi3v.py b/tests/models/decoder_only/vision_language/test_phi3v.py
index eba0a1a1bce4..00c1b9975ef3 100644
--- a/tests/models/decoder_only/vision_language/test_phi3v.py
+++ b/tests/models/decoder_only/vision_language/test_phi3v.py
@@ -90,7 +90,7 @@ def run_test(
     # max_model_len should be greater than image_feature_size
     with vllm_runner(model,
                      max_model_len=4096,
-                     max_num_seqs=1,
+                     max_num_seqs=2,
                      dtype=dtype,
                      limit_mm_per_prompt={"image": mm_limit},
                      tensor_parallel_size=tensor_parallel_size,
diff --git a/tests/models/decoder_only/vision_language/test_qwen.py b/tests/models/decoder_only/vision_language/test_qwen.py
index 638fb68b8f87..d2d0c62f5b2c 100644
--- a/tests/models/decoder_only/vision_language/test_qwen.py
+++ b/tests/models/decoder_only/vision_language/test_qwen.py
@@ -221,7 +221,7 @@ def run_test(
     # Qwen encodes each image into a fixed content size of 256
     with vllm_runner(model,
                      max_model_len=1024,
-                     max_num_seqs=1,
+                     max_num_seqs=2,
                      dtype=dtype,
                      limit_mm_per_prompt={"image": mm_limit},
                      tensor_parallel_size=tensor_parallel_size,

From 6a22c510a0da023231d5ee9fe16d8db51eee4cbf Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 01:06:49 +0000
Subject: [PATCH 06/31] Update examples to be based on test config

---
 examples/offline_inference_vision_language.py | 28 +++++++++++++------
 ...e_inference_vision_language_multi_image.py | 13 +++++++--
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/examples/offline_inference_vision_language.py b/examples/offline_inference_vision_language.py
index 6d34621a8a9b..adddd6bc3b2d 100644
--- a/examples/offline_inference_vision_language.py
+++ b/examples/offline_inference_vision_language.py
@@ -12,6 +12,10 @@
 from vllm.assets.video import VideoAsset
 from vllm.utils import FlexibleArgumentParser
 
+# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
+# lower-end GPUs.
+# Unless specified, these settings have been tested to work on a single L4.
+
 
 # LLaVA-1.5
 def run_llava(question, modality):
@@ -19,7 +23,7 @@ def run_llava(question, modality):
 
     prompt = f"USER: <image>\n{question}\nASSISTANT:"
 
-    llm = LLM(model="llava-hf/llava-1.5-7b-hf")
+    llm = LLM(model="llava-hf/llava-1.5-7b-hf", max_model_len=4096)
     stop_token_ids = None
     return llm, prompt, stop_token_ids
 
@@ -57,7 +61,7 @@ def run_llava_onevision(question, modality):
         <|im_start|>assistant\n"
 
     llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
-              max_model_len=32768)
+              max_model_len=16384)
     stop_token_ids = None
     return llm, prompt, stop_token_ids
 
@@ -67,7 +71,7 @@ def run_fuyu(question, modality):
     assert modality == "image"
 
     prompt = f"{question}\n"
-    llm = LLM(model="adept/fuyu-8b")
+    llm = LLM(model="adept/fuyu-8b", max_model_len=2560, max_num_seqs=2)
     stop_token_ids = None
     return llm, prompt, stop_token_ids
 
@@ -99,7 +103,8 @@ def run_phi3v(question, modality):
     llm = LLM(
         model="microsoft/Phi-3-vision-128k-instruct",
         trust_remote_code=True,
-        max_num_seqs=5,
+        max_model_len=4096,
+        max_num_seqs=2,
         mm_processor_kwargs={"num_crops": 16},
     )
     stop_token_ids = None
@@ -122,7 +127,7 @@ def run_chameleon(question, modality):
     assert modality == "image"
 
     prompt = f"{question}<image>"
-    llm = LLM(model="facebook/chameleon-7b")
+    llm = LLM(model="facebook/chameleon-7b", max_model_len=4096)
     stop_token_ids = None
     return llm, prompt, stop_token_ids
 
@@ -145,6 +150,8 @@ def run_minicpmv(question, modality):
                                               trust_remote_code=True)
     llm = LLM(
         model=model_name,
+        max_model_len=4096,
+        max_num_seqs=2,
         trust_remote_code=True,
     )
     # NOTE The stop_token_ids are different for various versions of MiniCPM-V
@@ -177,7 +184,7 @@ def run_internvl(question, modality):
     llm = LLM(
         model=model_name,
         trust_remote_code=True,
-        max_num_seqs=5,
+        max_model_len=4096,
     )
 
     tokenizer = AutoTokenizer.from_pretrained(model_name,
@@ -215,7 +222,8 @@ def run_qwen_vl(question, modality):
     llm = LLM(
         model="Qwen/Qwen-VL",
         trust_remote_code=True,
-        max_num_seqs=5,
+        max_model_len=1024,
+        max_num_seqs=2,
     )
 
     prompt = f"{question}Picture 1: <img></img>\n"
@@ -229,8 +237,10 @@ def run_qwen2_vl(question, modality):
 
     model_name = "Qwen/Qwen2-VL-7B-Instruct"
 
+    # Tested on L40
     llm = LLM(
         model=model_name,
+        max_model_len=8192,
         max_num_seqs=5,
     )
 
@@ -252,10 +262,10 @@ def run_mllama(question, modality):
     # max_model_len (131072) for this model may cause OOM.
     # You may lower either to run this example on lower-end GPUs.
 
-    # The configuration below has been confirmed to launch on a
-    # single H100 GPU.
+    # The configuration below has been confirmed to launch on a single L40 GPU.
     llm = LLM(
         model=model_name,
+        max_model_len=4096,
         max_num_seqs=16,
         enforce_eager=True,
     )
diff --git a/examples/offline_inference_vision_language_multi_image.py b/examples/offline_inference_vision_language_multi_image.py
index 8c5f1a7b7af0..1e99c02234d0 100644
--- a/examples/offline_inference_vision_language_multi_image.py
+++ b/examples/offline_inference_vision_language_multi_image.py
@@ -28,12 +28,18 @@ class ModelRequestData(NamedTuple):
     chat_template: Optional[str]
 
 
+# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
+# lower-end GPUs.
+# Unless specified, these settings have been tested to work on a single L4.
+
+
 def load_qwenvl_chat(question: str, image_urls: List[str]) -> ModelRequestData:
     model_name = "Qwen/Qwen-VL-Chat"
     llm = LLM(
         model=model_name,
         trust_remote_code=True,
-        max_num_seqs=5,
+        max_model_len=1024,
+        max_num_seqs=2,
         limit_mm_per_prompt={"image": len(image_urls)},
     )
     placeholders = "".join(f"Picture {i}: <img></img>\n"
@@ -83,6 +89,7 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
         model="microsoft/Phi-3.5-vision-instruct",
         trust_remote_code=True,
         max_model_len=4096,
+        max_num_seqs=2,
         limit_mm_per_prompt={"image": len(image_urls)},
         mm_processor_kwargs={"num_crops": 4},
     )
@@ -106,7 +113,6 @@ def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
     llm = LLM(
         model=model_name,
         trust_remote_code=True,
-        max_num_seqs=5,
         max_model_len=4096,
         limit_mm_per_prompt={"image": len(image_urls)},
     )
@@ -148,10 +154,11 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
 
     model_name = "Qwen/Qwen2-VL-7B-Instruct"
 
+    # Tested on L40
     llm = LLM(
         model=model_name,
-        max_num_seqs=5,
         max_model_len=32768 if process_vision_info is None else 4096,
+        max_num_seqs=5,
         limit_mm_per_prompt={"image": len(image_urls)},
     )
 

From 578f55eaa8465441297169b266b4b0de549ef793 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 03:24:14 +0000
Subject: [PATCH 07/31] Fix w8a8 for devices other than AMD GPU

---
 vllm/model_executor/layers/quantization/utils/w8a8_utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py
index fb263d121fe5..a6c6980bf901 100644
--- a/vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+++ b/vllm/model_executor/layers/quantization/utils/w8a8_utils.py
@@ -159,7 +159,8 @@ def apply_fp8_linear(
 
             # Making sure the dummy tensor is on the same device as the weight
             global TORCH_DEVICE_IDENTITY
-            if TORCH_DEVICE_IDENTITY.device != weight.device:
+            if (TORCH_DEVICE_IDENTITY is not None
+                    and TORCH_DEVICE_IDENTITY.device != weight.device):
                 TORCH_DEVICE_IDENTITY = TORCH_DEVICE_IDENTITY.to(weight.device)
 
             # GEMM

From f5d46dcec2ea64168673205aa45b9340dde769ba Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 03:24:33 +0000
Subject: [PATCH 08/31] Use quantized mllama model for testing to avoid OOM

---
 .../vision_language/test_mllama.py            | 143 ++++++++++--------
 1 file changed, 84 insertions(+), 59 deletions(-)

diff --git a/tests/models/encoder_decoder/vision_language/test_mllama.py b/tests/models/encoder_decoder/vision_language/test_mllama.py
index daec00720367..a6f44de02b4f 100644
--- a/tests/models/encoder_decoder/vision_language/test_mllama.py
+++ b/tests/models/encoder_decoder/vision_language/test_mllama.py
@@ -54,6 +54,39 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
     return hf_output_ids, hf_output_str, out_logprobs
 
 
+def _get_inputs(
+    image_assets: _ImageAssets,
+    *,
+    size_factors: Optional[List[float]] = None,
+    sizes: Optional[List[Tuple[int, int]]] = None,
+) -> List[Tuple[List[str], PromptImageInput]]:
+    images = [asset.pil_image for asset in image_assets]
+
+    if size_factors is not None:
+        inputs_per_image = [(
+            [prompt for _ in size_factors],
+            [rescale_image_size(image, factor) for factor in size_factors],
+        ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
+    elif sizes is not None:
+        inputs_per_image = [(
+            [
+                prompt if size is not None else text_only_prompts[0]
+                for size in sizes
+            ],
+            [
+                image.resize(size) if size is not None else None
+                for size in sizes
+            ],
+        ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
+        if len(sizes) == 0:
+            inputs_per_image.append(
+                (text_only_prompts, [None] * len(text_only_prompts)))
+    else:
+        raise ValueError("You must provide either `size_factors` or `sizes`")
+
+    return inputs_per_image
+
+
 @overload
 def run_test(
     hf_runner: Type[HfRunner],
@@ -102,39 +135,17 @@ def run_test(
     tensor_parallel_size: int,
     distributed_executor_backend: Optional[str] = None,
 ):
-    images = [asset.pil_image for asset in image_assets]
-
-    if size_factors is not None:
-        inputs_per_image = [(
-            [prompt for _ in size_factors],
-            [rescale_image_size(image, factor) for factor in size_factors],
-        ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
-    elif sizes is not None:
-        inputs_per_image = [(
-            [
-                prompt if size is not None else text_only_prompts[0]
-                for size in sizes
-            ],
-            [
-                image.resize(size) if size is not None else None
-                for size in sizes
-            ],
-        ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
-        if len(sizes) == 0:
-            inputs_per_image.append(
-                (text_only_prompts, [None] * len(text_only_prompts)))
-    else:
-        raise ValueError("You must provide either `size_factors` or `sizes`")
-
-    _run_test(hf_runner,
-              vllm_runner,
-              inputs_per_image,
-              model,
-              dtype=dtype,
-              max_tokens=max_tokens,
-              num_logprobs=num_logprobs,
-              tensor_parallel_size=tensor_parallel_size,
-              distributed_executor_backend=distributed_executor_backend)
+    _run_test(
+        hf_runner,
+        vllm_runner,
+        _get_inputs(image_assets, size_factors=size_factors, sizes=sizes),
+        model,
+        dtype=dtype,
+        max_tokens=max_tokens,
+        num_logprobs=num_logprobs,
+        tensor_parallel_size=tensor_parallel_size,
+        distributed_executor_backend=distributed_executor_backend,
+    )
 
 
 def _run_test(
@@ -216,26 +227,29 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
+SIZES = [
+    # Text only
+    [],
+    # Single-size
+    [(512, 512)],
+    # Single-size, batched
+    [(512, 512), (512, 512), (512, 512)],
+    # Multi-size, batched
+    [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
+     (1024, 1024), (512, 1536), (512, 2028)],
+    # Multi-size, batched, including text only
+    [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
+     (1024, 1024), (512, 1536), (512, 2028), None],
+    # mllama has 8 possible aspect ratios, carefully set the sizes
+    # to cover all of them
+]
+
+
+@pytest.mark.skip(
+    reason=
+    "Model is too big, test passed on L40 locally but will OOM on CI machine.")
 @pytest.mark.parametrize("model", models)
-@pytest.mark.parametrize(
-    "sizes",
-    [
-        # Text only
-        [],
-        # Single-size
-        [(512, 512)],
-        # Single-size, batched
-        [(512, 512), (512, 512), (512, 512)],
-        # Multi-size, batched
-        [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
-         (1024, 1024), (512, 1536), (512, 2028)],
-        # Multi-size, batched, including text only
-        [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
-         (1024, 1024), (512, 1536), (512, 2028), None],
-        # mllama has 8 possible aspect ratios, carefully set the sizes
-        # to cover all of them
-    ],
-)
+@pytest.mark.parametrize("sizes", SIZES)
 @pytest.mark.parametrize("dtype", ["bfloat16"])
 @pytest.mark.parametrize("max_tokens", [128])
 @pytest.mark.parametrize("num_logprobs", [5])
@@ -254,16 +268,27 @@ def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
     )
 
 
+@pytest.mark.parametrize("model", [
+    "neuralmagic/Llama-3.2-11B-Vision-Instruct-FP8-dynamic",
+])
+@pytest.mark.parametrize("sizes", SIZES)
+@pytest.mark.parametrize("max_tokens", [128])
+def test_quant_model(vllm_runner, image_assets, model, sizes, max_tokens):
+    with vllm_runner(model,
+                     dtype="auto",
+                     max_model_len=4096,
+                     max_num_seqs=2,
+                     enforce_eager=True,
+                     limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
+                                          }) as vllm_model:
+        for prompts, images in _get_inputs(image_assets, sizes=sizes):
+            vllm_model.generate_greedy(prompts, max_tokens, images=images)
+
+
 @multi_gpu_test(num_gpus=2)
 @pytest.mark.skip("ps.get_tp_group().all_gather() used in vLLM but not in HF")
 @pytest.mark.parametrize("model", models)
-@pytest.mark.parametrize(
-    "sizes",
-    [
-        [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
-         (1024, 1024), (512, 1536), (512, 2028), None],
-    ],
-)
+@pytest.mark.parametrize("sizes", [SIZES[-1]])
 @pytest.mark.parametrize("dtype", ["bfloat16"])
 @pytest.mark.parametrize("max_tokens", [128])
 @pytest.mark.parametrize("num_logprobs", [5])

From 6cea788820afed20e92cf3902456e2f6a644cb67 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 03:34:42 +0000
Subject: [PATCH 09/31] Fuyu test fails on `max_num_seqs=2`

---
 examples/offline_inference_vision_language.py          | 2 +-
 tests/models/decoder_only/vision_language/test_fuyu.py | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/examples/offline_inference_vision_language.py b/examples/offline_inference_vision_language.py
index adddd6bc3b2d..b94ef537d783 100644
--- a/examples/offline_inference_vision_language.py
+++ b/examples/offline_inference_vision_language.py
@@ -71,7 +71,7 @@ def run_fuyu(question, modality):
     assert modality == "image"
 
     prompt = f"{question}\n"
-    llm = LLM(model="adept/fuyu-8b", max_model_len=2560, max_num_seqs=2)
+    llm = LLM(model="adept/fuyu-8b", max_model_len=2048, max_num_seqs=2)
     stop_token_ids = None
     return llm, prompt, stop_token_ids
 
diff --git a/tests/models/decoder_only/vision_language/test_fuyu.py b/tests/models/decoder_only/vision_language/test_fuyu.py
index 3ec36cb003cd..236a7e30851c 100644
--- a/tests/models/decoder_only/vision_language/test_fuyu.py
+++ b/tests/models/decoder_only/vision_language/test_fuyu.py
@@ -65,8 +65,9 @@ def run_test(
 
     # max_model_len should be greater than image_feature_size
     with vllm_runner(model,
-                     max_model_len=2560,
-                     max_num_seqs=2,
+                     max_model_len=2048,
+                     # Fails on max_num_seqs=2
+                     max_num_seqs=1,
                      dtype=dtype,
                      tensor_parallel_size=tensor_parallel_size,
                      distributed_executor_backend=distributed_executor_backend,
@@ -80,8 +81,6 @@ def run_test(
         ]
 
     with hf_runner(model, dtype=dtype) as hf_model:
-        hf_model.model.get_output_embeddings = lambda: \
-            hf_model.model.language_model.get_output_embeddings()
         eos_token_id = hf_model.processor.tokenizer.eos_token_id
         hf_outputs_per_image = [
             hf_model.generate_greedy_logprobs_limit(prompts,

From da669d467cc680c360c91bcd233db67269821ae8 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 03:46:01 +0000
Subject: [PATCH 10/31] format

---
 .../decoder_only/vision_language/test_fuyu.py    | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/models/decoder_only/vision_language/test_fuyu.py b/tests/models/decoder_only/vision_language/test_fuyu.py
index 236a7e30851c..bed2687313ff 100644
--- a/tests/models/decoder_only/vision_language/test_fuyu.py
+++ b/tests/models/decoder_only/vision_language/test_fuyu.py
@@ -64,14 +64,14 @@ def run_test(
     # will hurt multiprocessing backend with fork method (the default method).
 
     # max_model_len should be greater than image_feature_size
-    with vllm_runner(model,
-                     max_model_len=2048,
-                     # Fails on max_num_seqs=2
-                     max_num_seqs=1,
-                     dtype=dtype,
-                     tensor_parallel_size=tensor_parallel_size,
-                     distributed_executor_backend=distributed_executor_backend,
-                     enforce_eager=True) as vllm_model:
+    with vllm_runner(
+            model,
+            max_model_len=2048,
+            max_num_seqs=1,  # Fails on max_num_seqs=2
+            dtype=dtype,
+            tensor_parallel_size=tensor_parallel_size,
+            distributed_executor_backend=distributed_executor_backend,
+            enforce_eager=True) as vllm_model:
         vllm_outputs_per_image = [
             vllm_model.generate_greedy_logprobs(prompts,
                                                 max_tokens,

From a9acefa3d79d000b16890f3db76395fb1fde19c1 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 03:46:15 +0000
Subject: [PATCH 11/31] Add fixme

---
 tests/models/decoder_only/vision_language/test_fuyu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/models/decoder_only/vision_language/test_fuyu.py b/tests/models/decoder_only/vision_language/test_fuyu.py
index bed2687313ff..4ce76981075f 100644
--- a/tests/models/decoder_only/vision_language/test_fuyu.py
+++ b/tests/models/decoder_only/vision_language/test_fuyu.py
@@ -67,7 +67,7 @@ def run_test(
     with vllm_runner(
             model,
             max_model_len=2048,
-            max_num_seqs=1,  # Fails on max_num_seqs=2
+            max_num_seqs=1,  # FIXME: Fails on max_num_seqs=2
             dtype=dtype,
             tensor_parallel_size=tensor_parallel_size,
             distributed_executor_backend=distributed_executor_backend,

From 9c88e93512ffd93fe8b51a3c4cfd68747aba9ca5 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 03:59:47 +0000
Subject: [PATCH 12/31] Add custom model tests

---
 .buildkite/test-pipeline.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 76df4a4fa9b4..a85edd3a6efc 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -329,6 +329,19 @@ steps:
     - pytest -v -s models/encoder_decoder/language
     - pytest -v -s models/encoder_decoder/vision_language
 
+- label: Custom Models Test
+  #mirror_hardwares: [amd]
+  source_file_dependencies:
+  # This step should always be run manually. No files should ever match the dependencies.
+  - NEVER
+  commands:
+    # PR authors can temporarily add commands below to test individual models
+    # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
+    # *To limit CI cost, remember to REMOVE them before merging the PR*
+    - pytest -v -s models/decoder_only/vision_language/test_llava_onevision.py
+    - pytest -v -s models/decoder_only/vision_language/test_fuyu.py
+    - pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
+
 #####  1 GPU test  #####
 #####  multi gpus test  #####
 

From 6cf7b2b583cff124e5d03382c90d18644b3bd550 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 04:00:16 +0000
Subject: [PATCH 13/31] Reword

---
 .buildkite/test-pipeline.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index a85edd3a6efc..4694c538b49c 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -337,7 +337,7 @@ steps:
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
-    # *To limit CI cost, remember to REMOVE them before merging the PR*
+    # *To avoid merge conflicts, remember to REMOVE them before merging the PR*
     - pytest -v -s models/decoder_only/vision_language/test_llava_onevision.py
     - pytest -v -s models/decoder_only/vision_language/test_fuyu.py
     - pytest -v -s models/encoder_decoder/vision_language/test_mllama.py

From 25879622b4f60de7f316812b52b695b5c69fb107 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 05:13:13 +0000
Subject: [PATCH 14/31] Reduce strictness

---
 .../models/decoder_only/vision_language/test_llava_onevision.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index 6852244574a7..9693814b83ab 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -214,7 +214,7 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
 )
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])
-@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize("num_logprobs", [10])
 @pytest.mark.parametrize("num_frames", [16])
 def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
                             dtype, max_tokens, num_logprobs,

From be7836d6c74c9a03f79769754813089de4776888 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 05:26:25 +0000
Subject: [PATCH 15/31] Test mllama distributed

---
 .buildkite/test-pipeline.yaml                               | 6 +++---
 tests/models/encoder_decoder/vision_language/test_mllama.py | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 4694c538b49c..9bf92342e2cc 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -331,9 +331,9 @@ steps:
 
 - label: Custom Models Test
   #mirror_hardwares: [amd]
-  source_file_dependencies:
-  # This step should always be run manually. No files should ever match the dependencies.
-  - NEVER
+  num_gpus: 2
+  # This step should always be run manually. No files should ever match these dependencies.
+  source_file_dependencies: [NEVER]
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
diff --git a/tests/models/encoder_decoder/vision_language/test_mllama.py b/tests/models/encoder_decoder/vision_language/test_mllama.py
index a6f44de02b4f..b1daee4bd672 100644
--- a/tests/models/encoder_decoder/vision_language/test_mllama.py
+++ b/tests/models/encoder_decoder/vision_language/test_mllama.py
@@ -286,7 +286,6 @@ def test_quant_model(vllm_runner, image_assets, model, sizes, max_tokens):
 
 
 @multi_gpu_test(num_gpus=2)
-@pytest.mark.skip("ps.get_tp_group().all_gather() used in vLLM but not in HF")
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize("sizes", [SIZES[-1]])
 @pytest.mark.parametrize("dtype", ["bfloat16"])

From 103f45c0284670e14cca7be281723595d5c29e5b Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 06:54:16 +0000
Subject: [PATCH 16/31] Document and use `optional` flag

---
 .buildkite/test-pipeline.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 9bf92342e2cc..239adfae9e48 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -9,6 +9,7 @@
 # label(str): the name of the test. emoji allowed.
 # fast_check(bool): whether to run this on each commit on fastcheck pipeline.
 # fast_check_only(bool): run this test on fastcheck pipeline only
+# optional(bool): never run this test by default (i.e. need to unblock manually)
 # command(str): the single command to run for tests. incompatible with commands.
 # commands(list): the list of commands to run for test. incompatbile with command.
 # mirror_hardwares(list): the list of hardwares to run the test on as well. currently only supports [amd]
@@ -332,12 +333,11 @@ steps:
 - label: Custom Models Test
   #mirror_hardwares: [amd]
   num_gpus: 2
-  # This step should always be run manually. No files should ever match these dependencies.
-  source_file_dependencies: [NEVER]
+  optional: true
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
-    # *To avoid merge conflicts, remember to REMOVE them before merging the PR*
+    # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
     - pytest -v -s models/decoder_only/vision_language/test_llava_onevision.py
     - pytest -v -s models/decoder_only/vision_language/test_fuyu.py
     - pytest -v -s models/encoder_decoder/vision_language/test_mllama.py

From 0dca6e1aee527c2c9f836de37d40711b59148a10 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 06:55:11 +0000
Subject: [PATCH 17/31] Fix unnecessary dtype upcasting

---
 tests/models/utils.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/models/utils.py b/tests/models/utils.py
index eb6254f18182..86a624483c58 100644
--- a/tests/models/utils.py
+++ b/tests/models/utils.py
@@ -1,9 +1,12 @@
 import warnings
 from typing import Dict, List, Optional, Sequence, Tuple, Union
 
+import torch
+
 from vllm.config import ModelConfig
 from vllm.inputs import InputContext
 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
+from vllm.utils import is_cpu
 
 TokensText = Tuple[List[int], str]
 
@@ -247,6 +250,7 @@ def check_logprobs_close(
 def build_model_context(model_name: str,
                         tokenizer_name: Optional[str] = None,
                         trust_remote_code: bool = False,
+                        dtype: Optional[Union[str, torch.dtype]] = None,
                         mm_processor_kwargs: Optional[Dict] = None,
                         limit_mm_per_prompt: Optional[Dict] = None):
     """Creates an InputContext for a given model.
@@ -264,12 +268,15 @@ def build_model_context(model_name: str,
     """
     if tokenizer_name is None:
         tokenizer_name = model_name
+    if dtype is None:
+        dtype = "bfloat16" if is_cpu() else "half"
+
     model_config = ModelConfig(
         model_name,
         tokenizer_name,
         tokenizer_mode="auto",
         trust_remote_code=trust_remote_code,
-        dtype="float32",
+        dtype=dtype,
         seed=0,
         mm_processor_kwargs=mm_processor_kwargs,
         limit_mm_per_prompt=limit_mm_per_prompt,

From 1ad1d49078be4e9ea8483c7bca61197a85f4abc2 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 06:56:17 +0000
Subject: [PATCH 18/31] Try a different prompt

---
 .../decoder_only/vision_language/test_llava_onevision.py      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index 9693814b83ab..fce86613da57 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -16,7 +16,7 @@
 # Video test
 HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
     "sample_demo_1":
-    "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n"  # noqa: E501
+    "<|im_start|>user\n<video>\nwhy is this video funny? Avoid mentioning any colors.<|im_end|>\n<|im_start|>assistant\n"  # noqa: E501
 })
 
 models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
@@ -214,7 +214,7 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
 )
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])
-@pytest.mark.parametrize("num_logprobs", [10])
+@pytest.mark.parametrize("num_logprobs", [5])
 @pytest.mark.parametrize("num_frames", [16])
 def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
                             dtype, max_tokens, num_logprobs,

From 1a738d4bd4e9df4769f82ed36a8553d8999741c5 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 08:10:14 +0000
Subject: [PATCH 19/31] Fix distributed test and remove quantized test

---
 .buildkite/test-pipeline.yaml                 |  9 +++--
 .../vision_language/test_broadcast.py         | 35 +++++++++++++++++
 .../vision_language/test_mllama.py            | 39 -------------------
 3 files changed, 40 insertions(+), 43 deletions(-)
 create mode 100644 tests/models/encoder_decoder/vision_language/test_broadcast.py

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 239adfae9e48..2a6002694b2d 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -332,15 +332,14 @@ steps:
 
 - label: Custom Models Test
   #mirror_hardwares: [amd]
+  working_dir: "/vllm-workspace/tests"
   num_gpus: 2
   optional: true
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
     # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
-    - pytest -v -s models/decoder_only/vision_language/test_llava_onevision.py
-    - pytest -v -s models/decoder_only/vision_language/test_fuyu.py
-    - pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
+    - pytest -v -s models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
 
 #####  1 GPU test  #####
 #####  multi gpus test  #####
@@ -390,7 +389,9 @@ steps:
   - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep -q 'Same node test passed'
   - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
   # Avoid importing model tests that cause CUDA reinitialization error
-  - pytest models/encoder_decoder/language/test_bart.py models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
+  - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
+  - pytest models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
+  - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
   - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
   - pip install -e ./plugins/vllm_add_dummy_model
   - pytest -v -s distributed/test_distributed_oot.py
diff --git a/tests/models/encoder_decoder/vision_language/test_broadcast.py b/tests/models/encoder_decoder/vision_language/test_broadcast.py
new file mode 100644
index 000000000000..542f41a38859
--- /dev/null
+++ b/tests/models/encoder_decoder/vision_language/test_broadcast.py
@@ -0,0 +1,35 @@
+import pytest
+
+from ....utils import multi_gpu_test
+
+
+@multi_gpu_test(num_gpus=2)
+@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
+@pytest.mark.parametrize("model", [
+    "meta-llama/Llama-3.2-11B-Vision-Instruct",
+])
+def test_models(hf_runner, vllm_runner, image_assets,
+                distributed_executor_backend, model) -> None:
+
+    dtype = "half"
+    max_tokens = 5
+    num_logprobs = 5
+    tensor_parallel_size = 2
+
+    if model.startswith("meta-llama/Llama-3.2-11B-Vision-Instruct"):
+        from .test_mllama import models, run_test
+    else:
+        raise NotImplementedError(f"Unsupported model: {model}")
+
+    run_test(
+        hf_runner,
+        vllm_runner,
+        image_assets,
+        model=models[0],
+        size_factors=[0.25, 0.5, 1.0],
+        dtype=dtype,
+        max_tokens=max_tokens,
+        num_logprobs=num_logprobs,
+        tensor_parallel_size=tensor_parallel_size,
+        distributed_executor_backend=distributed_executor_backend,
+    )
diff --git a/tests/models/encoder_decoder/vision_language/test_mllama.py b/tests/models/encoder_decoder/vision_language/test_mllama.py
index b1daee4bd672..2738ffa2da0d 100644
--- a/tests/models/encoder_decoder/vision_language/test_mllama.py
+++ b/tests/models/encoder_decoder/vision_language/test_mllama.py
@@ -9,7 +9,6 @@
 
 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
                           _ImageAssets)
-from ....utils import multi_gpu_test
 from ...utils import check_logprobs_close
 
 _LIMIT_IMAGE_PER_PROMPT = 1
@@ -266,41 +265,3 @@ def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
         num_logprobs=num_logprobs,
         tensor_parallel_size=1,
     )
-
-
-@pytest.mark.parametrize("model", [
-    "neuralmagic/Llama-3.2-11B-Vision-Instruct-FP8-dynamic",
-])
-@pytest.mark.parametrize("sizes", SIZES)
-@pytest.mark.parametrize("max_tokens", [128])
-def test_quant_model(vllm_runner, image_assets, model, sizes, max_tokens):
-    with vllm_runner(model,
-                     dtype="auto",
-                     max_model_len=4096,
-                     max_num_seqs=2,
-                     enforce_eager=True,
-                     limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
-                                          }) as vllm_model:
-        for prompts, images in _get_inputs(image_assets, sizes=sizes):
-            vllm_model.generate_greedy(prompts, max_tokens, images=images)
-
-
-@multi_gpu_test(num_gpus=2)
-@pytest.mark.parametrize("model", models)
-@pytest.mark.parametrize("sizes", [SIZES[-1]])
-@pytest.mark.parametrize("dtype", ["bfloat16"])
-@pytest.mark.parametrize("max_tokens", [128])
-@pytest.mark.parametrize("num_logprobs", [5])
-def test_models_distributed(hf_runner, vllm_runner, image_assets, model, sizes,
-                            dtype, max_tokens, num_logprobs) -> None:
-    run_test(
-        hf_runner,
-        vllm_runner,
-        image_assets,
-        model,
-        sizes=sizes,
-        dtype=dtype,
-        max_tokens=max_tokens,
-        num_logprobs=num_logprobs,
-        tensor_parallel_size=2,
-    )

From 11604f0061efce33db59a8c461fb71544544b3f3 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 10:13:56 +0000
Subject: [PATCH 20/31] Try 4 GPU test

---
 .buildkite/test-pipeline.yaml                               | 6 +++---
 .../encoder_decoder/vision_language/test_broadcast.py       | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 2a6002694b2d..17bb95cd3840 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -113,6 +113,7 @@ steps:
   commands:
   - pytest -v -s distributed/test_pynccl.py
   - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
+  - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_4_gpus
 
 - label: Metrics, Tracing Test # 10min
   num_gpus: 2 
@@ -333,13 +334,13 @@ steps:
 - label: Custom Models Test
   #mirror_hardwares: [amd]
   working_dir: "/vllm-workspace/tests"
-  num_gpus: 2
+  num_gpus: 4
   optional: true
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
     # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
-    - pytest -v -s models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
+    - pytest -v -s models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_4_gpus
 
 #####  1 GPU test  #####
 #####  multi gpus test  #####
@@ -391,7 +392,6 @@ steps:
   # Avoid importing model tests that cause CUDA reinitialization error
   - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
   - pytest models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
-  - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
   - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
   - pip install -e ./plugins/vllm_add_dummy_model
   - pytest -v -s distributed/test_distributed_oot.py
diff --git a/tests/models/encoder_decoder/vision_language/test_broadcast.py b/tests/models/encoder_decoder/vision_language/test_broadcast.py
index 542f41a38859..b11da65d3571 100644
--- a/tests/models/encoder_decoder/vision_language/test_broadcast.py
+++ b/tests/models/encoder_decoder/vision_language/test_broadcast.py
@@ -3,7 +3,7 @@
 from ....utils import multi_gpu_test
 
 
-@multi_gpu_test(num_gpus=2)
+@multi_gpu_test(num_gpus=4)
 @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
 @pytest.mark.parametrize("model", [
     "meta-llama/Llama-3.2-11B-Vision-Instruct",
@@ -14,7 +14,7 @@ def test_models(hf_runner, vllm_runner, image_assets,
     dtype = "half"
     max_tokens = 5
     num_logprobs = 5
-    tensor_parallel_size = 2
+    tensor_parallel_size = 4
 
     if model.startswith("meta-llama/Llama-3.2-11B-Vision-Instruct"):
         from .test_mllama import models, run_test

From 74ad70c9c2e3b47584ff976819f58ff4f9426a9a Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 12:05:22 +0000
Subject: [PATCH 21/31] Enable distributed inference in `HfRunner`, and revert
 to 2 GPU setup

---
 .buildkite/test-pipeline.yaml                 |  4 +-
 tests/conftest.py                             | 84 ++++++++++---------
 .../vision_language/test_broadcast.py         |  4 +-
 .../vision_language/test_mllama.py            |  1 +
 4 files changed, 49 insertions(+), 44 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 17bb95cd3840..8923d71a6f46 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -113,7 +113,6 @@ steps:
   commands:
   - pytest -v -s distributed/test_pynccl.py
   - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
-  - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_4_gpus
 
 - label: Metrics, Tracing Test # 10min
   num_gpus: 2 
@@ -340,7 +339,7 @@ steps:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
     # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
-    - pytest -v -s models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_4_gpus
+    - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
 
 #####  1 GPU test  #####
 #####  multi gpus test  #####
@@ -391,6 +390,7 @@ steps:
   - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m distributed_2_gpus
   # Avoid importing model tests that cause CUDA reinitialization error
   - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
+  - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
   - pytest models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
   - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
   - pip install -e ./plugins/vllm_add_dummy_model
diff --git a/tests/conftest.py b/tests/conftest.py
index db71d8bc3af1..45dc5e8323ca 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -246,17 +246,14 @@ def video_assets() -> _VideoAssets:
 
 class HfRunner:
 
-    def wrap_device(self, input: _T) -> _T:
-        if not is_cpu():
-            # Check if the input is already on the GPU
-            if hasattr(input, 'device') and input.device.type == "cuda":
-                return input  # Already on GPU, no need to move
-            return input.to("cuda")
-        else:
-            # Check if the input is already on the CPU
-            if hasattr(input, 'device') and input.device.type == "cpu":
-                return input  # Already on CPU, no need to move
-            return input.to("cpu")
+    def wrap_device(self, input: _T, device: Optional[str] = None) -> _T:
+        if device is None:
+            return self.wrap_device(input, "cpu" if is_cpu() else "cuda")
+
+        if hasattr(input, "device") and input.device.type == device:
+            return input
+
+        return input.to(device)
 
     def __init__(
         self,
@@ -333,7 +330,7 @@ def generate(
             inputs = self.postprocess_inputs(inputs)
 
             output_ids = self.model.generate(
-                **self.wrap_device(inputs),
+                **self.wrap_device(inputs, device=self.model.device.type),
                 use_cache=True,
                 **kwargs,
             )
@@ -406,7 +403,7 @@ def generate_greedy_logprobs(
             inputs = self.postprocess_inputs(inputs)
 
             output = self.model.generate(
-                **self.wrap_device(inputs),
+                **self.wrap_device(inputs, device=self.model.device.type),
                 use_cache=True,
                 do_sample=False,
                 max_new_tokens=max_tokens,
@@ -414,40 +411,39 @@ def generate_greedy_logprobs(
                 return_dict_in_generate=True,
                 **kwargs,
             )
-            seq_logprobs: List[torch.Tensor] = []
-            for hidden_states in output.hidden_states:
-                last_hidden_states = hidden_states[-1][0]
-                logits = torch.matmul(
-                    last_hidden_states,
-                    self.model.get_output_embeddings().weight.t(),
-                )
-                if self.model.get_output_embeddings().bias is not None:
-                    logits += self.model.get_output_embeddings(
-                    ).bias.unsqueeze(0)
-                logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
-                seq_logprobs.append(logprobs)
+            seq_logprobs = self._hidden_states_to_seq_logprobs(
+                output.hidden_states)
             all_logprobs.append(seq_logprobs)
         return all_logprobs
 
-    def _hidden_states_to_logprobs(
+    def _hidden_states_to_seq_logprobs(
         self,
-        hidden_states,
-        num_logprobs,
-    ) -> Tuple[List[Dict[int, float]], int]:
+        hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],
+    ) -> List[torch.Tensor]:
+        output_embeddings = self.model.get_output_embeddings()
+
         seq_logprobs: List[torch.Tensor] = []
-        output_len = len(hidden_states)
         for _, hidden_state in enumerate(hidden_states):
             last_hidden_states = hidden_state[-1][0]
             logits = torch.matmul(
-                last_hidden_states,
-                self.model.get_output_embeddings().weight.t(),
+                last_hidden_states.to(output_embeddings.weight.device),
+                output_embeddings.weight.t(),
             )
-            if getattr(self.model.get_output_embeddings(), "bias",
-                       None) is not None:
-                logits += self.model.get_output_embeddings().bias.unsqueeze(0)
+            if getattr(output_embeddings, "bias", None) is not None:
+                logits += output_embeddings.bias.unsqueeze(0)
             logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
             seq_logprobs.append(logprobs)
 
+        return seq_logprobs
+
+    def _hidden_states_to_logprobs(
+        self,
+        hidden_states: Tuple[Tuple[torch.Tensor, ...], ...],
+        num_logprobs: int,
+    ) -> Tuple[List[Dict[int, float]], int]:
+        seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)
+        output_len = len(hidden_states)
+
         # convert to dict
         seq_logprobs_lst: List[Dict[int, float]] = []
         for tok_idx, tok_logprobs in enumerate(seq_logprobs):
@@ -500,7 +496,7 @@ def generate_greedy_logprobs_limit(
             inputs = self.postprocess_inputs(inputs)
 
             output = self.model.generate(
-                **self.wrap_device(inputs),
+                **self.wrap_device(inputs, device=self.model.device.type),
                 use_cache=True,
                 do_sample=False,
                 max_new_tokens=max_tokens,
@@ -543,12 +539,20 @@ def generate_encoder_decoder_greedy_logprobs_limit(
 
         for (encoder_prompt,
              decoder_prompt) in to_enc_dec_tuple_list(encoder_decoder_prompts):
+
             encoder_input_ids = self.wrap_device(
-                self.tokenizer(encoder_prompt, return_tensors="pt").input_ids)
-            decoder_input_ids = (
-                None if decoder_prompt is None else self.wrap_device(
+                self.tokenizer(encoder_prompt, return_tensors="pt").input_ids,
+                device=self.model.device.type,
+            )
+
+            if decoder_prompt is None:
+                decoder_input_ids = None
+            else:
+                decoder_input_ids = self.wrap_device(
                     self.tokenizer(decoder_prompt,
-                                   return_tensors="pt").input_ids))
+                                   return_tensors="pt").input_ids,
+                    device=self.model.device.type,
+                )
 
             output = self.model.generate(
                 encoder_input_ids,
diff --git a/tests/models/encoder_decoder/vision_language/test_broadcast.py b/tests/models/encoder_decoder/vision_language/test_broadcast.py
index b11da65d3571..542f41a38859 100644
--- a/tests/models/encoder_decoder/vision_language/test_broadcast.py
+++ b/tests/models/encoder_decoder/vision_language/test_broadcast.py
@@ -3,7 +3,7 @@
 from ....utils import multi_gpu_test
 
 
-@multi_gpu_test(num_gpus=4)
+@multi_gpu_test(num_gpus=2)
 @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
 @pytest.mark.parametrize("model", [
     "meta-llama/Llama-3.2-11B-Vision-Instruct",
@@ -14,7 +14,7 @@ def test_models(hf_runner, vllm_runner, image_assets,
     dtype = "half"
     max_tokens = 5
     num_logprobs = 5
-    tensor_parallel_size = 4
+    tensor_parallel_size = 2
 
     if model.startswith("meta-llama/Llama-3.2-11B-Vision-Instruct"):
         from .test_mllama import models, run_test
diff --git a/tests/models/encoder_decoder/vision_language/test_mllama.py b/tests/models/encoder_decoder/vision_language/test_mllama.py
index 2738ffa2da0d..ea09b758afc8 100644
--- a/tests/models/encoder_decoder/vision_language/test_mllama.py
+++ b/tests/models/encoder_decoder/vision_language/test_mllama.py
@@ -201,6 +201,7 @@ def process(hf_inputs: BatchEncoding):
     AutoConfig.register("mllama", MllamaConfigHf, exist_ok=True)
     with hf_runner(model,
                    dtype=dtype,
+                   model_kwargs={"device_map": "auto"},
                    postprocess_inputs=process,
                    auto_cls=AutoModelForVision2Seq) as hf_model:
         hf_outputs_per_image = [

From 30d2ea0571d79c562b64ad9864b5e54a0b7b8368 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 12:57:21 +0000
Subject: [PATCH 22/31] Clear out Custom Models Tests

---
 .buildkite/test-pipeline.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 8923d71a6f46..a3f5606b176e 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -332,14 +332,11 @@ steps:
 
 - label: Custom Models Test
   #mirror_hardwares: [amd]
-  working_dir: "/vllm-workspace/tests"
-  num_gpus: 4
   optional: true
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
     # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
-    - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
 
 #####  1 GPU test  #####
 #####  multi gpus test  #####

From 307ae15c61e380a23eb2cd7069bf0a0c6e668dc3 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Fri, 27 Sep 2024 13:00:45 +0000
Subject: [PATCH 23/31] Update recorded test durations

---
 .buildkite/test-pipeline.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index a3f5606b176e..d706f9e190f9 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -301,7 +301,7 @@ steps:
     - pytest -v -s models/test_oot_registration.py # it needs a clean process
     - pytest -v -s models/*.py --ignore=models/test_oot_registration.py
 
-- label: Decoder-only Language Models Test # 1h3min
+- label: Decoder-only Language Models Test # 1h30min
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/
@@ -309,7 +309,7 @@ steps:
   commands:
     - pytest -v -s models/decoder_only/language
 
-- label: Decoder-only Multi-Modal Models Test # 56min
+- label: Decoder-only Multi-Modal Models Test # 1h30min
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/
@@ -319,7 +319,7 @@ steps:
     - pytest -v -s models/decoder_only/audio_language
     - pytest -v -s models/decoder_only/vision_language
 
-- label: Other Models Test # 5min
+- label: Other Models Test # 10min
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/

From 56d7ff6db122880faaf64cae6168c17722ecf0be Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Fri, 27 Sep 2024 18:49:46 -0700
Subject: [PATCH 24/31] add source dependency and skip image

---
 .buildkite/test-pipeline.yaml                                   | 2 ++
 .../models/decoder_only/vision_language/test_llava_onevision.py | 1 +
 2 files changed, 3 insertions(+)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index d706f9e190f9..875ad13e836b 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -333,6 +333,8 @@ steps:
 - label: Custom Models Test
   #mirror_hardwares: [amd]
   optional: true
+  source_file_dependencies:
+  - tests/models/encoder_decoder/vision_language
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index fce86613da57..27d65d59a297 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -300,6 +300,7 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
+@pytest.skip()
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])

From 62d32078eb716905f04794dac4425c0d8ca696c2 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Fri, 27 Sep 2024 20:00:07 -0700
Subject: [PATCH 25/31] typo

---
 .../models/decoder_only/vision_language/test_llava_onevision.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index 27d65d59a297..398701707310 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -300,7 +300,7 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
-@pytest.skip()
+@pytest.mark.skip()
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])

From e012bd7b8d896e6fc21396639a18ac964f9112d1 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Sat, 28 Sep 2024 03:06:31 +0000
Subject: [PATCH 26/31] Fix location of dependencies

---
 .buildkite/test-pipeline.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 875ad13e836b..93e7045e9d7f 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -325,6 +325,7 @@ steps:
   - vllm/
   - tests/models/embedding/language
   - tests/models/encoder_decoder/language
+  - tests/models/encoder_decoder/vision_language
   commands:
     - pytest -v -s models/embedding/language
     - pytest -v -s models/encoder_decoder/language
@@ -333,8 +334,6 @@ steps:
 - label: Custom Models Test
   #mirror_hardwares: [amd]
   optional: true
-  source_file_dependencies:
-  - tests/models/encoder_decoder/vision_language
   commands:
     # PR authors can temporarily add commands below to test individual models
     # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py

From 0ce6b01f348384ccccf09351eb0cc90e25ebab2d Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Sat, 28 Sep 2024 03:07:31 +0000
Subject: [PATCH 27/31] Update timings

---
 .buildkite/test-pipeline.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 93e7045e9d7f..50990de6be07 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -301,7 +301,7 @@ steps:
     - pytest -v -s models/test_oot_registration.py # it needs a clean process
     - pytest -v -s models/*.py --ignore=models/test_oot_registration.py
 
-- label: Decoder-only Language Models Test # 1h30min
+- label: Decoder-only Language Models Test # 1h36min
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/
@@ -309,7 +309,7 @@ steps:
   commands:
     - pytest -v -s models/decoder_only/language
 
-- label: Decoder-only Multi-Modal Models Test # 1h30min
+- label: Decoder-only Multi-Modal Models Test # 1h31min
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/
@@ -319,7 +319,7 @@ steps:
     - pytest -v -s models/decoder_only/audio_language
     - pytest -v -s models/decoder_only/vision_language
 
-- label: Other Models Test # 10min
+- label: Other Models Test # 6min
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/

From 8f36fa1d5e6056d25cd9ec24cd6a1e291280589d Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Sat, 28 Sep 2024 03:11:57 +0000
Subject: [PATCH 28/31] More timing updates

---
 .buildkite/test-pipeline.yaml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index 50990de6be07..bb42b5f29a72 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -40,7 +40,7 @@ steps:
   # Check API reference (if it fails, you may have missing mock imports)
   - grep \"sig sig-object py\" build/html/dev/sampling_params.html
 
-- label: Async Engine, Inputs, Utils, Worker Test # 15min
+- label: Async Engine, Inputs, Utils, Worker Test # 24min
   fast_check: true
   source_file_dependencies:
   - vllm/
@@ -82,7 +82,7 @@ steps:
   commands:
   - pytest -v -s core
 
-- label: Entrypoints Test # 20min
+- label: Entrypoints Test # 40min
   working_dir: "/vllm-workspace/tests"
   fast_check: true
   mirror_hardwares: [amd]
@@ -152,7 +152,7 @@ steps:
   # OOM in the CI unless we run this separately
   - pytest -v -s tokenization
 
-- label: Examples Test # 12min
+- label: Examples Test # 15min
   working_dir: "/vllm-workspace/examples"
   #mirror_hardwares: [amd]
   source_file_dependencies:
@@ -170,7 +170,7 @@ steps:
     - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
     - python3 offline_inference_encoder_decoder.py
 
-- label: Prefix Caching Test # 7min
+- label: Prefix Caching Test # 9min
   #mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/
@@ -178,7 +178,7 @@ steps:
   commands:
     - pytest -v -s prefix_caching
 
-- label: Samplers Test # 18min
+- label: Samplers Test # 36min
   source_file_dependencies:
   - vllm/model_executor/layers
   - vllm/sampling_metadata.py
@@ -194,7 +194,7 @@ steps:
   - tests/test_logits_processor
   command: pytest -v -s test_logits_processor.py
 
-- label: Speculative decoding tests # 22min
+- label: Speculative decoding tests # 30min
   source_file_dependencies:
   - vllm/spec_decode
   - tests/spec_decode
@@ -204,7 +204,7 @@ steps:
     - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
     - pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
 
-- label: LoRA Test %N # 30min each
+- label: LoRA Test %N # 15min each
   mirror_hardwares: [amd]
   source_file_dependencies:
   - vllm/lora
@@ -212,7 +212,7 @@ steps:
   command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
   parallelism: 4
 
-- label: "PyTorch Fullgraph Smoke Test"
+- label: "PyTorch Fullgraph Smoke Test" # 9min
   fast_check: true
   source_file_dependencies:
   - vllm/
@@ -220,14 +220,14 @@ steps:
   commands:
   - pytest -v -s compile/test_full_graph_smoke.py
 
-- label: "PyTorch Fullgraph Test"
+- label: "PyTorch Fullgraph Test" # 18min
   source_file_dependencies:
   - vllm/
   - tests/compile
   commands:
   - pytest -v -s compile/test_full_graph.py
 
-- label: Kernels Test %N # 30min each
+- label: Kernels Test %N # 1h each
   mirror_hardwares: [amd]
   source_file_dependencies:
   - csrc/
@@ -257,7 +257,7 @@ steps:
   - pip install aiohttp
   - bash run-benchmarks.sh
 
-- label: Quantization Test # 15min
+- label: Quantization Test # 33min
   source_file_dependencies:
   - csrc/
   - vllm/model_executor/layers/quantization
@@ -370,7 +370,7 @@ steps:
   - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
     - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep -q 'Same node test passed'
 
-- label: Distributed Tests (2 GPUs) # 28min
+- label: Distributed Tests (2 GPUs) # 40min
   #mirror_hardwares: [amd]
   working_dir: "/vllm-workspace/tests"
   num_gpus: 2
@@ -396,7 +396,7 @@ steps:
   - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
   - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
 
-- label: Multi-step Tests (4 GPUs) # 21min
+- label: Multi-step Tests (4 GPUs) # 36min
   working_dir: "/vllm-workspace/tests"
   num_gpus: 4
   source_file_dependencies:
@@ -414,7 +414,7 @@ steps:
   - pytest -v -s multi_step/test_correctness_async_llm.py
   - pytest -v -s multi_step/test_correctness_llm.py
 
-- label: Pipeline Parallelism Test # 23min
+- label: Pipeline Parallelism Test # 45min
   working_dir: "/vllm-workspace/tests"
   num_gpus: 4
   source_file_dependencies:
@@ -440,7 +440,7 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - pytest -v -s -x lora/test_long_context.py
 
-- label: Weight Loading Multiple GPU Test
+- label: Weight Loading Multiple GPU Test  # 33min
   working_dir: "/vllm-workspace/tests"
   num_gpus: 2
   source_file_dependencies:

From bc078de7acccc9dd9a92667f97cf5ca948b83857 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Sat, 28 Sep 2024 05:37:32 +0000
Subject: [PATCH 29/31] Fix wrong test being skipped

---
 .../models/decoder_only/vision_language/test_llava_onevision.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index 398701707310..605775236fba 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -164,6 +164,7 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
+@pytest.mark.skip("Different outputs but still reasonable")
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize(
     "size_factors",
@@ -300,7 +301,6 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
-@pytest.mark.skip()
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])

From bd2400b0c78e33e8ddbd2ef3fc28887794334dcd Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Sat, 28 Sep 2024 05:39:46 +0000
Subject: [PATCH 30/31] Revert prompt extension since test still fails
 regardless

---
 .../models/decoder_only/vision_language/test_llava_onevision.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index 605775236fba..470c1dbcaf0c 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -16,7 +16,7 @@
 # Video test
 HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
     "sample_demo_1":
-    "<|im_start|>user\n<video>\nwhy is this video funny? Avoid mentioning any colors.<|im_end|>\n<|im_start|>assistant\n"  # noqa: E501
+    "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n"  # noqa: E501
 })
 
 models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]

From dc6cea7197550ba4c6ee091142a28161d4eae951 Mon Sep 17 00:00:00 2001
From: DarkLight1337 <tlleungac@connect.ust.hk>
Date: Sat, 28 Sep 2024 07:25:50 +0000
Subject: [PATCH 31/31] Use 7B model to avoid different output in
 LLaVA-OneVision; instead skip tests because OOM

---
 .../vision_language/test_llava_onevision.py          | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/models/decoder_only/vision_language/test_llava_onevision.py b/tests/models/decoder_only/vision_language/test_llava_onevision.py
index 470c1dbcaf0c..2c4cd3fb8529 100644
--- a/tests/models/decoder_only/vision_language/test_llava_onevision.py
+++ b/tests/models/decoder_only/vision_language/test_llava_onevision.py
@@ -19,7 +19,7 @@
     "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n"  # noqa: E501
 })
 
-models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
+models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
 
 
 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -164,7 +164,9 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
-@pytest.mark.skip("Different outputs but still reasonable")
+@pytest.mark.skip(
+    reason=
+    "Model is too big, test passed on L40 locally but will OOM on CI machine.")
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize(
     "size_factors",
@@ -208,6 +210,9 @@ def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
     )
 
 
+@pytest.mark.skip(
+    reason=
+    "Model is too big, test passed on L40 locally but will OOM on CI machine.")
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize(
     "sizes",
@@ -301,6 +306,9 @@ def process(hf_inputs: BatchEncoding):
         )
 
 
+@pytest.mark.skip(
+    reason=
+    "Model is too big, test passed on L40 locally but will OOM on CI machine.")
 @pytest.mark.parametrize("model", models)
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [128])