From 25d2f183a23c923e5b3b7810d5652be38899b3be Mon Sep 17 00:00:00 2001 From: Reese Wang Date: Tue, 26 Apr 2022 10:26:35 +0000 Subject: [PATCH 1/2] Revise CMake scripts for ninja-build --- cmake/external/dgc.cmake | 3 +-- cmake/external/gflags.cmake | 2 -- cmake/external/openblas.cmake | 2 +- cmake/external/rocksdb.cmake | 2 +- cmake/third_party.cmake | 3 +++ paddle/fluid/distributed/CMakeLists.txt | 14 +++++++++++--- paddle/fluid/distributed/common/CMakeLists.txt | 1 + 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cmake/external/dgc.cmake b/cmake/external/dgc.cmake index 711d6c5b10aacc..c0d1ea8a384a74 100644 --- a/cmake/external/dgc.cmake +++ b/cmake/external/dgc.cmake @@ -29,7 +29,7 @@ ExternalProject_Add( URL_MD5 "94e6fa1bc97169d0e1aad44570fe3251" PREFIX "${DGC_PREFIX_DIR}" CONFIGURE_COMMAND "" - BUILD_COMMAND make -j $(nproc) + BUILD_COMMAND make -j${NPROC} INSTALL_COMMAND mkdir -p ${DGC_INSTALL_DIR}/lib/ ${DGC_INCLUDE_DIR}/dgc && cp ${DGC_SOURCES_DIR}/build/lib/libdgc.a ${DGC_LIBRARIES} && cp ${DGC_SOURCES_DIR}/build/include/dgc.h ${DGC_INCLUDE_DIR}/dgc/ @@ -40,4 +40,3 @@ ExternalProject_Add( ADD_LIBRARY(dgc STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET dgc PROPERTY IMPORTED_LOCATION ${DGC_LIBRARIES}) ADD_DEPENDENCIES(dgc extern_dgc) - diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 056ff32c8c0d91..2a8f97c974a690 100755 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -23,8 +23,6 @@ IF(WIN32) set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) ELSE(WIN32) set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/libgflags.a" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) - set(BUILD_COMMAND $(MAKE) --silent) - set(INSTALL_COMMAND $(MAKE) install) ENDIF(WIN32) INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index b099831738599e..87356c59db63ae 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -51,7 +51,7 @@ IF(NOT WIN32) PREFIX ${CBLAS_PREFIX_DIR} INSTALL_DIR ${CBLAS_INSTALL_DIR} BUILD_IN_SOURCE 1 - BUILD_COMMAND make -j$(nproc) ${COMMON_ARGS} ${OPTIONAL_ARGS} + BUILD_COMMAND make -j${NPROC} ${COMMON_ARGS} ${OPTIONAL_ARGS} INSTALL_COMMAND make install NO_SHARED=1 NO_LAPACK=1 PREFIX= UPDATE_COMMAND "" CONFIGURE_COMMAND "" diff --git a/cmake/external/rocksdb.cmake b/cmake/external/rocksdb.cmake index befbc8138fc50f..ab46a65b67671b 100644 --- a/cmake/external/rocksdb.cmake +++ b/cmake/external/rocksdb.cmake @@ -38,6 +38,7 @@ ExternalProject_Add( INSTALL_COMMAND mkdir -p ${ROCKSDB_INSTALL_DIR}/lib/ && cp ${ROCKSDB_PREFIX_DIR}/src/extern_rocksdb/librocksdb.a ${ROCKSDB_LIBRARIES} && cp -r ${ROCKSDB_PREFIX_DIR}/src/extern_rocksdb/include ${ROCKSDB_INSTALL_DIR}/ + BYPRODUCTS ${ROCKSDB_LIBRARIES} BUILD_IN_SOURCE 1 ) @@ -48,4 +49,3 @@ SET_PROPERTY(TARGET rocksdb PROPERTY IMPORTED_LOCATION ${ROCKSDB_LIBRARIES}) ADD_DEPENDENCIES(rocksdb extern_rocksdb) LIST(APPEND external_project_dependencies rocksdb) - diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index eb6fa4ee13c81e..534a44ae57473e 100755 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -23,6 +23,9 @@ set(THIRD_PARTY_CACHE_PATH "${CMAKE_SOURCE_DIR}" CACHE STRING set(THIRD_PARTY_BUILD_TYPE Release) set(third_party_deps) +include(ProcessorCount) +ProcessorCount(NPROC) + # cache funciton to avoid repeat download code of third_party. # This function has 4 parameters, URL / REPOSITOR / TAG / DIR: # 1. URL: specify download url of 3rd party diff --git a/paddle/fluid/distributed/CMakeLists.txt b/paddle/fluid/distributed/CMakeLists.txt index 107a19cb7decc0..dfb111abbc2288 100755 --- a/paddle/fluid/distributed/CMakeLists.txt +++ b/paddle/fluid/distributed/CMakeLists.txt @@ -22,9 +22,17 @@ if(NOT WITH_PSCORE) endif() proto_library(ps_framework_proto SRCS the_one_ps.proto) -add_custom_command(TARGET ps_framework_proto POST_BUILD - COMMAND mv the_one_ps.pb.h ps.pb.h - COMMAND mv the_one_ps.pb.cc ps.pb.cc) +add_custom_target( + ps.pb.h + COMMAND cp the_one_ps.pb.h ps.pb.h + DEPENDS ps_framework_proto +) + +add_custom_target( + ps.pb.cc + COMMAND cp the_one_ps.pb.cc ps.pb.cc + DEPENDS ps_framework_proto +) set(DISTRIBUTE_COMPILE_FLAGS "-Wno-error=unused-value -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=type-limits -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result") diff --git a/paddle/fluid/distributed/common/CMakeLists.txt b/paddle/fluid/distributed/common/CMakeLists.txt index eab6165ca689e1..9f7c9d1df1e9bd 100644 --- a/paddle/fluid/distributed/common/CMakeLists.txt +++ b/paddle/fluid/distributed/common/CMakeLists.txt @@ -1,4 +1,5 @@ cc_library(afs_wrapper SRCS afs_warpper.cc DEPS fs ps_framework_proto) +add_dependencies(afs_wrapper ps.pb.h ps.pb.cc) #set_property(GLOBAL PROPERTY COMMON_DEPS afs_warpper) From 4ea407c73ff6c544f2af683aaee4241da83f3364 Mon Sep 17 00:00:00 2001 From: Reese Wang Date: Sat, 14 May 2022 12:19:56 +0000 Subject: [PATCH 2/2] Add ps.pb.h dependency --- paddle/fluid/distributed/ps/table/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/fluid/distributed/ps/table/CMakeLists.txt b/paddle/fluid/distributed/ps/table/CMakeLists.txt index f2b9eb71f5a640..8372168637e820 100644 --- a/paddle/fluid/distributed/ps/table/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/table/CMakeLists.txt @@ -40,7 +40,9 @@ set_source_files_properties(ssd_sparse_table.cc PROPERTIES COMPILE_FLAGS ${DISTR set_source_files_properties(memory_sparse_geo_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) cc_library(sparse_sgd_rule SRCS sparse_sgd_rule.cc DEPS ${TABLE_DEPS} ps_framework_proto) +add_dependencies(sparse_sgd_rule ps.pb.h ps.pb.cc) cc_library(ctr_accessor SRCS ctr_accessor.cc ctr_double_accessor.cc sparse_accessor.cc DEPS ${TABLE_DEPS} ps_framework_proto sparse_sgd_rule) +add_dependencies(ctr_accessor ps.pb.h ps.pb.cc) cc_library(sparse_table SRCS memory_sparse_table.cc ssd_sparse_table.cc memory_sparse_geo_table.cc DEPS ps_framework_proto ${TABLE_DEPS} fs afs_wrapper ctr_accessor common_table rocksdb) cc_library(table SRCS table.cc DEPS sparse_table common_table tensor_accessor tensor_table ps_framework_proto string_helper device_context gflags glog boost)