Dynamic scheduler delay to improve ITL performance

tests/core/test_scheduler.py

@@ -1,5 +1,6 @@
 from typing import List
 import pytest  # noqa
+import time
 
 from vllm.config import CacheConfig, SchedulerConfig
 from vllm.core.scheduler import Scheduler
@@ -168,3 +169,36 @@ def test_scheduler_max_seqs():
     # and one is prompting.
     _, out = scheduler.schedule()
     assert set(out.scheduled_seq_groups) == set([all_seq_groups[1]])
+
+
+def test_scheduler_use_delay():
+
+    block_size = 4
+    scheduler_config = SchedulerConfig(100, 64, 16, use_delay=True)
+    cache_config = CacheConfig(block_size, 1.0, 1, "auto")
+    cache_config.num_cpu_blocks = 8
+    cache_config.num_gpu_blocks = 8
+    scheduler = Scheduler(scheduler_config, cache_config, None)
+
+    # schedule first prompt
+    _, seq_group = create_dummy_prompt("0", prompt_length=block_size)
+    scheduler.add_seq_group(seq_group)
+    seq_group_meta, out = scheduler.schedule()
+    assert out.prompt_run
+    assert seq_group_meta[0].request_id == '0'
+
+    # wait for a second before scheduling next prompt
+    time.sleep(1)
+    _, seq_group = create_dummy_prompt("1", prompt_length=block_size)
+    scheduler.add_seq_group(seq_group)
+
+    # second prompt should *not* be scheduled
+    seq_group_meta, out = scheduler.schedule()
+    assert not out.prompt_run
+    assert seq_group_meta[0].request_id == '0'
+
+    # wait for more than 0.5 second and try again
+    time.sleep(0.6)
+    seq_group_meta, out = scheduler.schedule()
+    assert out.prompt_run
+    assert seq_group_meta[0].request_id == '1'

vllm/config.py

@@ -535,13 +535,16 @@ class SchedulerConfig:
             iteration.
         max_model_len: Maximum length of a sequence (including prompt
             and generated text).
+        use_delay: Apply artificial scheduling delay equal to half of previous
+            prefill time to allow the waiting queue to fill up.
     """
 
     def __init__(
         self,
         max_num_batched_tokens: Optional[int],
         max_num_seqs: int,
         max_model_len: int,
+        use_delay: bool = False,
     ) -> None:
         if max_num_batched_tokens is not None:
             self.max_num_batched_tokens = max_num_batched_tokens
@@ -551,6 +554,7 @@ def __init__(
             self.max_num_batched_tokens = max(max_model_len, 2048)
         self.max_num_seqs = max_num_seqs
         self.max_model_len = max_model_len
+        self.use_delay = use_delay
         self._verify_args()
 
     def _verify_args(self) -> None:

vllm/core/scheduler.py

@@ -102,6 +102,9 @@ def __init__(
         self.running: Deque[SequenceGroup] = deque()
         # Sequence groups in the SWAPPED state.
         self.swapped: Deque[SequenceGroup] = deque()
+        self.prev_time = 0.0
+        self.prev_prompt = False
+        self.last_prompt_latency = 0.0
 
     @property
     def lora_enabled(self) -> bool:
@@ -179,7 +182,7 @@ def _schedule(self) -> SchedulerOutputs:
             # are added to the back.
             leftover_waiting_sequences = deque()
             num_batched_tokens = 0
-            while self.waiting:
+            while self._passed_delay(now) and self.waiting:
                 seq_group = self.waiting[0]
                 waiting_seqs = seq_group.get_seqs(
                     status=SequenceStatus.WAITING)
@@ -246,6 +249,7 @@ def _schedule(self) -> SchedulerOutputs:
             self.waiting.extendleft(leftover_waiting_sequences)
 
             if scheduled or ignored_seq_groups:
+                self.prev_prompt = True
                 scheduler_outputs = SchedulerOutputs(
                     scheduled_seq_groups=scheduled,
                     prompt_run=True,
@@ -491,3 +495,18 @@ def _swap_out(
 
     def mark_blocks_as_computed(self, seq_group: SequenceGroup):
         self.block_manager.mark_blocks_as_computed(seq_group)
+
+    def _passed_delay(self, now: float) -> bool:
+        if self.prev_prompt:
+            self.last_prompt_latency = now - self.prev_time
+        self.prev_time, self.prev_prompt = now, False
+        # Delay scheduling prompts to let waiting queue fill up
+        if self.scheduler_config.use_delay and self.waiting:
+            earliest_arrival_time = min(
+                [e.metrics.arrival_time for e in self.waiting])
+            passed_delay = ((now - earliest_arrival_time) >
+                            (0.5 * self.last_prompt_latency)
+                            or not self.running)
+        else:
+            passed_delay = True
+        return passed_delay

vllm/engine/arg_utils.py

@@ -51,6 +51,7 @@ class EngineArgs:
     max_cpu_loras: Optional[int] = None
     device: str = 'auto'
     ray_workers_use_nsight: bool = False
+    scheduler_use_delay: bool = False
 
     def __post_init__(self):
         if self.tokenizer is None:
@@ -305,6 +306,11 @@ def add_cli_args(
                             default=EngineArgs.device,
                             choices=["auto", "cuda", "neuron"],
                             help='Device type for vLLM execution.')
+        parser.add_argument(
+            '--scheduler-use-delay',
+            action='store_true',
+            help='Apply artificial scheduling delay equal to half of '
+            'previous prompt time.')
         return parser
 
     @classmethod
@@ -342,7 +348,8 @@ def create_engine_configs(
             ), self.ray_workers_use_nsight)
         scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
                                            self.max_num_seqs,
-                                           model_config.max_model_len)
+                                           model_config.max_model_len,
+                                           self.scheduler_use_delay)
         lora_config = LoRAConfig(
             max_lora_rank=self.max_lora_rank,
             max_loras=self.max_loras,