From a571750b251abf624bb62cd092d8a5861b1945d8 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 15 Mar 2025 10:50:58 -0700 Subject: [PATCH 1/5] [BugFix] Fix torch distributed stateless PG backend init ProcessGroup.Options was removed in torch 2.6: https://github.com/pytorch/pytorch/pull/132931 Signed-off-by: Nick Hill --- vllm/distributed/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index 25202062e975..d045106e5fb0 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -299,15 +299,14 @@ def stateless_init_torch_distributed_process_group( # different systems (e.g. RPC) in case the store is multi-tenant. prefix_store = PrefixStore(init_method, store) - pg_options = ProcessGroup.Options(backend=backend, timeout=timeout) - pg: ProcessGroup = ProcessGroup( prefix_store, group_rank, group_size, - pg_options, ) + pg._set_default_backend(backend) + if backend == "gloo": from torch.distributed.distributed_c10d import ProcessGroupGloo backend_class = ProcessGroupGloo(prefix_store, @@ -327,6 +326,8 @@ def stateless_init_torch_distributed_process_group( backend_options) backend_type = ProcessGroup.BackendType.NCCL device = torch.device("cuda") + else: + raise RuntimeError(f"Unsupported backend type: {backend}") backend_class._set_sequence_number_for_group() From 8cdc7145d8b932af4f035ac288e69c8b3f4ca09b Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 15 Mar 2025 10:56:52 -0700 Subject: [PATCH 2/5] Ensure DP test fails properly Signed-off-by: Nick Hill --- examples/offline_inference/data_parallel.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/offline_inference/data_parallel.py b/examples/offline_inference/data_parallel.py index b00519314d8b..e395edcb43f9 100644 --- a/examples/offline_inference/data_parallel.py +++ b/examples/offline_inference/data_parallel.py @@ -73,8 +73,10 @@ def main(dp_size, dp_rank, dp_master_ip, dp_master_port, GPUs_per_dp_rank): for i in range(DP_size): proc = Process(target=main, args=(DP_size, i, dp_master_ip, dp_master_port, - GPUs_per_dp_rank)) + GPUs_per_dp_rank), daemon=True) proc.start() procs.append(proc) for proc in procs: proc.join() + if proc.exitcode: + exit(proc.exitcode) From 9bbae737586967fbf5a4bd45cd211d42a95deab1 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 15 Mar 2025 11:01:03 -0700 Subject: [PATCH 3/5] fix linting Signed-off-by: Nick Hill --- examples/offline_inference/data_parallel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/offline_inference/data_parallel.py b/examples/offline_inference/data_parallel.py index e395edcb43f9..31b27a81faf6 100644 --- a/examples/offline_inference/data_parallel.py +++ b/examples/offline_inference/data_parallel.py @@ -73,7 +73,8 @@ def main(dp_size, dp_rank, dp_master_ip, dp_master_port, GPUs_per_dp_rank): for i in range(DP_size): proc = Process(target=main, args=(DP_size, i, dp_master_ip, dp_master_port, - GPUs_per_dp_rank), daemon=True) + GPUs_per_dp_rank), + daemon=True) proc.start() procs.append(proc) for proc in procs: From 4c5ed2d3ef146f284f4d59631a4f2c323e299815 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 15 Mar 2025 14:26:19 -0700 Subject: [PATCH 4/5] fix default backend call Signed-off-by: Nick Hill --- vllm/distributed/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/utils.py b/vllm/distributed/utils.py index d045106e5fb0..84899358a6d6 100644 --- a/vllm/distributed/utils.py +++ b/vllm/distributed/utils.py @@ -305,8 +305,6 @@ def stateless_init_torch_distributed_process_group( group_size, ) - pg._set_default_backend(backend) - if backend == "gloo": from torch.distributed.distributed_c10d import ProcessGroupGloo backend_class = ProcessGroupGloo(prefix_store, @@ -327,8 +325,9 @@ def stateless_init_torch_distributed_process_group( backend_type = ProcessGroup.BackendType.NCCL device = torch.device("cuda") else: - raise RuntimeError(f"Unsupported backend type: {backend}") + raise RuntimeError(f"Unsupported torch distributed backend: {backend}") + pg._set_default_backend(backend_type) backend_class._set_sequence_number_for_group() pg._register_backend(device, backend_type, backend_class) From 1a97d369305776333e2a64b0b9a6614f5a31049f Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 15 Mar 2025 15:46:35 -0700 Subject: [PATCH 5/5] fix daemon proc in data_parallel.py Signed-off-by: Nick Hill --- examples/offline_inference/data_parallel.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/offline_inference/data_parallel.py b/examples/offline_inference/data_parallel.py index 31b27a81faf6..b73770ce382c 100644 --- a/examples/offline_inference/data_parallel.py +++ b/examples/offline_inference/data_parallel.py @@ -73,11 +73,13 @@ def main(dp_size, dp_rank, dp_master_ip, dp_master_port, GPUs_per_dp_rank): for i in range(DP_size): proc = Process(target=main, args=(DP_size, i, dp_master_ip, dp_master_port, - GPUs_per_dp_rank), - daemon=True) + GPUs_per_dp_rank)) proc.start() procs.append(proc) + exit_code = 0 for proc in procs: proc.join() if proc.exitcode: - exit(proc.exitcode) + exit_code = proc.exitcode + + exit(exit_code)