Merge pull request #185 from nikitaxgusev/17_release

Intel(R) oneAPI Collective Communications Library (oneCCL) 2021.17

Author: Maria1Petrova

CMakeLists.txt

@@ -75,6 +75,7 @@ option(ENABLE_UMF "Enable UMF support" TRUE)
 option(ENABLE_STUB_BACKEND "Enable stub backend" TRUE)
 option(ENABLE_LINKER_RUNPATH "Enable linker runpath flags" FALSE)
 option(ENABLE_OMP "Enable openMP extension for intra-node collectives" TRUE)
+option(ENABLE_SYCL_PER_KERNEL_COMPILE "Enable SYCL device code module assembly per kernel" TRUE)
 
 option(USE_CODECOV_FLAGS "Calculate code coverage" FALSE)
 option(WITH_ASAN "Use address sanitizer, can only be used in Debug build" FALSE)
@@ -107,6 +108,7 @@ message(STATUS "Enable DRM support: ${ENABLE_DRM}")
 message(STATUS "Enable stub backend: ${ENABLE_STUB_BACKEND}")
 message(STATUS "Enable linker rpath flags: ${ENABLE_LINKER_RUNPATH}")
 message(STATUS "Enable openMP extension for intra-node collectives: ${ENABLE_OMP}")
+message(STATUS "Enable SYCL device code module assembly per kernel: ${ENABLE_SYCL_PER_KERNEL_COMPILE}")
 
 add_definitions(-DCCL_C_COMPILER="${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
 add_definitions(-DCCL_CXX_COMPILER="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
@@ -227,6 +229,12 @@ if (COMPUTE_BACKEND)
     endif()
     if (${COMPUTE_BACKEND} STREQUAL "dpcpp" AND ${CMAKE_CXX_COMPILER} MATCHES ".*icpx")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl")
+        if (ENABLE_SYCL_PER_KERNEL_COMPILE AND "${CMAKE_BUILD_TYPE_CASE_INSENSITIVE}" STREQUAL "release")
+            message(STATUS "Enabling SYCL device code module assembly per kernel")
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-device-code-split=per_kernel")
+        else()
+            message(STATUS "Skipping per-kernel SYCL split for ${CMAKE_BUILD_TYPE}")
+        endif()
     endif()
 endif()
 
@@ -342,8 +350,8 @@ file(GLOB spv_kernels "${PROJECT_SOURCE_DIR}/src/kernels/kernels.spv")
 endif()
 
 set(CCL_MAJOR_VERSION     "2021")
-set(CCL_MINOR_VERSION     "16")
-set(CCL_UPDATE_VERSION    "2")
+set(CCL_MINOR_VERSION     "17")
+set(CCL_UPDATE_VERSION    "0")
 set(CCL_PRODUCT_STATUS    "Gold")
 string(TIMESTAMP CCL_PRODUCT_BUILD_DATE "%Y-%m-%dT %H:%M:%SZ")
 get_vcs_properties("git")

cmake/ccl

@@ -69,4 +69,4 @@ setenv             CCL_ROOT                             "$topdir"
 
 prepend-path       LD_LIBRARY_PATH                      "$topdir/lib"
 prepend-path       LIBRARY_PATH                         "$topdir/lib"
-prepend-path       CPATH                                "$topdir/include"
+prepend-path       CPLUS_INCLUDE_PATH                   "$topdir/include"

cmake/setvars.sh.in

@@ -108,7 +108,8 @@ fi
 LD_LIBRARY_PATH=$(prepend_path "${I_MPI_ROOT}/libfabric/lib" "${LD_LIBRARY_PATH:-}") ; export LD_LIBRARY_PATH
 FI_PROVIDER_PATH="${I_MPI_ROOT}/libfabric/lib/prov:/usr/lib64/libfabric"; export FI_PROVIDER_PATH
 
-CPATH=$(prepend_path "${I_MPI_ROOT}/include" "${CPATH:-}"); export CPATH
+C_INCLUDE_PATH=$(prepend_path "${I_MPI_ROOT}/include" "${C_INCLUDE_PATH:-}"); export C_INCLUDE_PATH
+CPLUS_INCLUDE_PATH=$(prepend_path "${I_MPI_ROOT}/include" "${CPLUS_INCLUDE_PATH:-}"); export CPLUS_INCLUDE_PATH
 LD_LIBRARY_PATH=$(prepend_path "${I_MPI_ROOT}/lib" "${LD_LIBRARY_PATH:-}") ; export LD_LIBRARY_PATH
 LIBRARY_PATH=$(prepend_path "${I_MPI_ROOT}/lib" "${LIBRARY_PATH:-}"); export LIBRARY_PATH
 PATH="${I_MPI_ROOT}/bin:${PATH}"; export PATH

cmake/vars.sh.in

@@ -110,7 +110,7 @@ WORK_DIR=$(get_script_path "${vars_script_name:-}")
 
 CCL_ROOT="$(cd "${WORK_DIR}"/../; pwd -P)"; export CCL_ROOT
 
-CPATH=$(prepend_path "${CCL_ROOT}/include" "${CPATH:-}"); export CPATH
+CPLUS_INCLUDE_PATH=$(prepend_path "${CCL_ROOT}/include" "${CPLUS_INCLUDE_PATH:-}"); export CPLUS_INCLUDE_PATH
 CMAKE_PREFIX_PATH=$(prepend_path "${CCL_ROOT}/lib/cmake/oneCCL" "${CMAKE_PREFIX_PATH:-}"); export CMAKE_PREFIX_PATH
 LIBRARY_PATH=$(prepend_path "${CCL_ROOT}/lib" "${LIBRARY_PATH:-}"); export LIBRARY_PATH
 LD_LIBRARY_PATH=$(prepend_path "${CCL_ROOT}/lib" "${LD_LIBRARY_PATH:-}"); export LD_LIBRARY_PATH
@@ -136,7 +136,8 @@ if [ -z "${SETVARS_CALL:-}" ] ; then
 
     if [ "$ccl_bundled_mpi" = "yes" ] ; then
         export I_MPI_ROOT="${CCL_ROOT}/opt/mpi"
-        CPATH=$(prepend_path "${I_MPI_ROOT}/include" "${CPATH:-}"); export CPATH
+        C_INCLUDE_PATH=$(prepend_path "${I_MPI_ROOT}/include" "${C_INCLUDE_PATH:-}"); export C_INCLUDE_PATH
+        CPLUS_INCLUDE_PATH=$(prepend_path "${I_MPI_ROOT}/include" "${CPLUS_INCLUDE_PATH:-}"); export CPLUS_INCLUDE_PATH
         LD_LIBRARY_PATH=$(prepend_path "${I_MPI_ROOT}/lib" "${LD_LIBRARY_PATH:-}") ; export LD_LIBRARY_PATH
         LIBRARY_PATH=$(prepend_path "${I_MPI_ROOT}/lib" "${LIBRARY_PATH:-}"); export LIBRARY_PATH
         PATH="${I_MPI_ROOT}/bin:${PATH}"; export PATH

deps/hwloc/include/hwloc.h

@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009 CNRS
- * Copyright © 2009-2024 Inria.  All rights reserved.
+ * Copyright © 2009-2025 Inria.  All rights reserved.
  * Copyright © 2009-2012 Université Bordeaux
  * Copyright © 2009-2020 Cisco Systems, Inc.  All rights reserved.
  * See COPYING in top-level directory.
@@ -112,7 +112,7 @@ extern "C" {
  * Two stable releases of the same series usually have the same ::HWLOC_API_VERSION
  * even if their HWLOC_VERSION are different.
  */
-#define HWLOC_API_VERSION 0x00020b00
+#define HWLOC_API_VERSION 0x00020c00
 
 /** \brief Indicate at runtime which hwloc API version was used at build time.
  *
@@ -346,9 +346,10 @@ typedef enum {
 			 *
 			 * Some operating systems (e.g. Linux) may expose a single die per package
 			 * even if the hardware does not support dies at all. To avoid showing
-			 * such non-existing dies, the corresponding hwloc backend may filter them out.
+			 * such non-existing dies, hwloc will filter them out if all of them are
+                         * identical to packages.
 			 * This is functionally equivalent to ::HWLOC_TYPE_FILTER_KEEP_STRUCTURE
-			 * being enforced.
+			 * being enforced for Dies versus Packages.
 			 */
 
   HWLOC_OBJ_TYPE_MAX    /**< \private Sentinel value */
@@ -1047,7 +1048,7 @@ HWLOC_DECLSPEC const char * hwloc_obj_type_string (hwloc_obj_type_t type) __hwlo
  * If \p size is 0, \p string may safely be \c NULL.
  *
  * \return the number of characters that were actually written if not truncating,
- * or that would have been written (not including the ending \\0).
+ * or that would have been written (not including the ending \c \0).
  */
 HWLOC_DECLSPEC int hwloc_obj_type_snprintf(char * __hwloc_restrict string, size_t size,
 					   hwloc_obj_t obj,
@@ -1062,7 +1063,7 @@ HWLOC_DECLSPEC int hwloc_obj_type_snprintf(char * __hwloc_restrict string, size_
  * If \p size is 0, \p string may safely be \c NULL.
  *
  * \return the number of characters that were actually written if not truncating,
- * or that would have been written (not including the ending \\0).
+ * or that would have been written (not including the ending \c \0).
  */
 HWLOC_DECLSPEC int hwloc_obj_attr_snprintf(char * __hwloc_restrict string, size_t size,
 					   hwloc_obj_t obj, const char * __hwloc_restrict separator,
@@ -2002,7 +2003,7 @@ HWLOC_DECLSPEC int hwloc_topology_set_xml(hwloc_topology_t __hwloc_restrict topo
  * a file, as with hwloc_topology_set_xml()).
  *
  * Gather topology information from the XML memory buffer given at
- * \p buffer and of length \p size (including an ending \0).
+ * \p buffer and of length \p size (including an ending \c \0).
  * This buffer may have been filled earlier with
  * hwloc_topology_export_xmlbuffer() in hwloc/export.h.
  *

deps/hwloc/include/hwloc/autogen/config.h

@@ -12,10 +12,10 @@
 #ifndef HWLOC_CONFIG_H
 #define HWLOC_CONFIG_H
 
-#define HWLOC_VERSION "2.11.2rc2-git"
+#define HWLOC_VERSION "2.12.0rc2-git"
 #define HWLOC_VERSION_MAJOR 2
-#define HWLOC_VERSION_MINOR 11
-#define HWLOC_VERSION_RELEASE 2
+#define HWLOC_VERSION_MINOR 12
+#define HWLOC_VERSION_RELEASE 0
 #define HWLOC_VERSION_GREEK "rc2"
 
 /* #undef HWLOC_PCI_COMPONENT_BUILTIN */

deps/hwloc/include/hwloc/bitmap.h

@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009 CNRS
- * Copyright © 2009-2023 Inria.  All rights reserved.
+ * Copyright © 2009-2024 Inria.  All rights reserved.
  * Copyright © 2009-2012 Université Bordeaux
  * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
  * See COPYING in top-level directory.
@@ -113,77 +113,132 @@ HWLOC_DECLSPEC int hwloc_bitmap_copy(hwloc_bitmap_t dst, hwloc_const_bitmap_t sr
  * Bitmap/String Conversion
  */
 
-/** \brief Stringify a bitmap.
+/** \brief Stringify a bitmap in the default hwloc format.
+ *
+ * <b>Note that if the bitmap is a CPU or nodeset, it contains physical indexes.</b>
+ *
+ * Print the bits set inside a bitmap as a comma-separated list of hexadecimal 32-bit blocks.
+ * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as <tt>"0xffffffff,0x00000006,0x00000002"</tt>.
  *
  * Up to \p buflen characters may be written in buffer \p buf.
  *
  * If \p buflen is 0, \p buf may safely be \c NULL.
  *
  * \return the number of characters that were actually written if not truncating,
- * or that would have been written (not including the ending \\0).
+ * or that would have been written (not including the ending \c \0).
+ * \return -1 on error.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap);
 
-/** \brief Stringify a bitmap into a newly allocated string.
+/** \brief Stringify a bitmap into a newly allocated string in the default hwloc format.
  *
- * \return 0 on success, -1 on error.
+ * <b>Note that if the bitmap is a CPU or nodeset, it contains physical indexes.</b>
+ *
+ * Print the bits set inside a bitmap as a comma-separated list of hexadecimal 32-bit blocks.
+ * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as <tt>"0xffffffff,0x00000006,0x00000002"</tt>.
+ *
+ * \return the number of characters that were written (not including the ending \c \0).
+ * \return -1 on error, for instance with \p errno set to \c ENOMEM on failure to allocate the output string.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_asprintf(char ** strp, hwloc_const_bitmap_t bitmap);
 
-/** \brief Parse a bitmap string and stores it in bitmap \p bitmap.
+/** \brief Parse a bitmap string as the default hwloc format and stores it in bitmap \p bitmap.
+ *
+ * <b>Note that if the bitmap is a CPU or nodeset, the input string must contain physical indexes.</b>
+ *
+ * The input string should be a comma-separared list of hexadecimal 32-bit blocks.
+ * String <tt>"0xffffffff,0x6,0x2"</tt> is parsed as a bitmap containing all bits between 64 and 95,
+ * and bits 33, 34 and 1.
  *
  * \return 0 on success, -1 on error.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_sscanf(hwloc_bitmap_t bitmap, const char * __hwloc_restrict string);
 
 /** \brief Stringify a bitmap in the list format.
+ *
+ * <b>Note that if the bitmap is a CPU or nodeset, it contains physical indexes.</b>
  *
  * Lists are comma-separated indexes or ranges.
  * Ranges are dash separated indexes.
- * The last range may not have an ending indexes if the bitmap is infinitely set.
+ * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as <tt>"1,33-34,64-95"</tt>.
+ * The last range may not have an ending index if the bitmap is infinitely set.
  *
  * Up to \p buflen characters may be written in buffer \p buf.
  *
  * If \p buflen is 0, \p buf may safely be \c NULL.
  *
  * \return the number of characters that were actually written if not truncating,
- * or that would have been written (not including the ending \\0).
+ * or that would have been written (not including the ending \c \0).
+ * \return -1 on error.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_list_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap);
 
 /** \brief Stringify a bitmap into a newly allocated list string.
  *
- * \return 0 on success, -1 on error.
+ * <b>Note that if the bitmap is a CPU or nodeset, it contains physical indexes.</b>
+ *
+ * Lists are comma-separated indexes or ranges.
+ * Ranges are dash separated indexes.
+ * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as <tt>"1,33-34,64-95"</tt>.
+ * The last range may not have an ending index if the bitmap is infinitely set.
+ *
+ * \return the number of characters that were written (not including the ending \c \0).
+ * \return -1 on error, for instance with \p errno set to \c ENOMEM on failure to allocate the output string.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_list_asprintf(char ** strp, hwloc_const_bitmap_t bitmap);
 
 /** \brief Parse a list string and stores it in bitmap \p bitmap.
+ *
+ * <b>Note that if the bitmap is a CPU or nodeset, the input string must contain physical indexes.</b>
+ *
+ * Lists are comma-separated indexes or ranges.
+ * Ranges are dash separated indexes.
+ * String <tt>"1,33-34,64-95"</tt> is parsed as a bitmap containing bits 1, 33, 34, and all from 64 to 95.
+ * The last range may not have an ending index if the bitmap is infinitely set.
  *
  * \return 0 on success, -1 on error.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_list_sscanf(hwloc_bitmap_t bitmap, const char * __hwloc_restrict string);
 
 /** \brief Stringify a bitmap in the taskset-specific format.
  *
- * The taskset command manipulates bitmap strings that contain a single
+ * <b>Note that if the bitmap is a CPU or nodeset, it contains physical indexes.</b>
+ *
+ * The taskset program manipulates bitmap strings that contain a single
  * (possible very long) hexadecimal number starting with 0x.
+ * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as </tt>"0xffffffff0000000600000002"</tt>.
  *
  * Up to \p buflen characters may be written in buffer \p buf.
  *
  * If \p buflen is 0, \p buf may safely be \c NULL.
  *
  * \return the number of characters that were actually written if not truncating,
- * or that would have been written (not including the ending \\0).
+ * or that would have been written (not including the ending \c \0).
+ * \return -1 on error.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap);
 
 /** \brief Stringify a bitmap into a newly allocated taskset-specific string.
  *
- * \return 0 on success, -1 on error.
+ * <b>Note that if the bitmap is a CPU or nodeset, it contains physical indexes.</b>
+ *
+ * The taskset program manipulates bitmap strings that contain a single
+ * (possible very long) hexadecimal number starting with 0x.
+ * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as <tt>"0xffffffff0000000600000002"</tt>.
+ *
+ * \return the number of characters that were written (not including the ending \c \0).
+ * \return -1 on error, for instance with \p errno set to \c ENOMEM on failure to allocate the output string.
  */
 HWLOC_DECLSPEC int hwloc_bitmap_taskset_asprintf(char ** strp, hwloc_const_bitmap_t bitmap);
 
 /** \brief Parse a taskset-specific bitmap string and stores it in bitmap \p bitmap.
+ *
+ * <b>Note that if the bitmap is a CPU or nodeset, the input string must contain physical indexes.</b>
+ *
+ * The taskset program manipulates bitmap strings that contain a single
+ * (possible very long) hexadecimal number starting with 0x.
+ * String <tt>"0xffffffff0000000600000002"</tt> is parsed as a bitmap containing all bits between 64 and 95,
+ * and bits 33, 34 and 1.
  *
  * \return 0 on success, -1 on error.
  */

deps/hwloc/include/hwloc/diff.h

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2013-2023 Inria.  All rights reserved.
+ * Copyright © 2013-2024 Inria.  All rights reserved.
  * See COPYING in top-level directory.
  */
 
@@ -258,7 +258,7 @@ HWLOC_DECLSPEC int hwloc_topology_diff_export_xml(hwloc_topology_diff_t diff, co
 /** \brief Load a list of topology differences from a XML buffer.
  *
  * Build a list of differences from the XML memory buffer given
- * at \p xmlbuffer and of length \p buflen (including an ending \0).
+ * at \p xmlbuffer and of length \p buflen (including an ending \c \0).
  * This buffer may have been filled earlier with
  * hwloc_topology_diff_export_xmlbuffer().
  *
@@ -284,7 +284,7 @@ HWLOC_DECLSPEC int hwloc_topology_diff_load_xmlbuffer(const char *xmlbuffer, int
  * that contains the reference topology.
  * This attribute is given back when reading the diff from XML.
  *
- * The returned buffer ends with a \0 that is included in the returned
+ * The returned buffer ends with a \c \0 that is included in the returned
  * length.
  *
  * \return 0 on success, -1 on error.

deps/hwloc/include/hwloc/distances.h

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2010-2024 Inria.  All rights reserved.
+ * Copyright © 2010-2025 Inria.  All rights reserved.
  * See COPYING in top-level directory.
  */
 
@@ -227,17 +227,24 @@ enum hwloc_distances_transform_e {
   HWLOC_DISTANCES_TRANSFORM_LINKS = 1,
 
   /** \brief Merge switches with multiple ports into a single object.
-   * This currently only applies to NVSwitches where GPUs seem connected to different
-   * separate switch ports in the NVLinkBandwidth matrix. This transformation will
-   * replace all of them with the same port connected to all GPUs.
-   * Other ports are removed by applying ::HWLOC_DISTANCES_TRANSFORM_REMOVE_NULL internally.
+   *
+   * This currently only applies to NVSwitches where GPUs seem connected
+   * to different switch ports. Switch ports must be objects with subtype
+   * "NVSwitch" as in the NVLinkBandwidth matrix.
+   *
+   * This transformation will replace all ports with only the first one,
+   * now connected to all GPUs. Other ports are removed by applying
+   * ::HWLOC_DISTANCES_TRANSFORM_REMOVE_NULL internally.
    * \hideinitializer
    */
   HWLOC_DISTANCES_TRANSFORM_MERGE_SWITCH_PORTS = 2,
 
   /** \brief Apply a transitive closure to the matrix to connect objects across switches.
-   * This currently only applies to GPUs and NVSwitches in the NVLinkBandwidth matrix.
-   * All pairs of GPUs will be reported as directly connected.
+   *
+   * All pairs of GPUs will be reported as directly connected instead GPUs being
+   * only connected to switches.
+   *
+   * Switch ports must be objects with subtype "NVSwitch" as in the NVLinkBandwidth matrix.
    * \hideinitializer
    */
   HWLOC_DISTANCES_TRANSFORM_TRANSITIVE_CLOSURE = 3