Skip to content

Commit fbbc339

Browse files
ThunderbrookThunderbrook
authored
[pslib] pslib with cmake (#32800)
* pslib with cmake * heter util * vlog * heter server test * add dtor * cmake
1 parent 5aa8faa commit fbbc339

19 files changed

Lines changed: 403 additions & 312 deletions

‎cmake/third_party.cmake‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,14 @@ if(WITH_PSLIB)
261261
if(WITH_PSLIB_BRPC)
262262
include(external/pslib_brpc) # download, build, install pslib_brpc
263263
list(APPEND third_party_deps extern_pslib_brpc)
264+
else()
265+
include(external/snappy)
266+
list(APPEND third_party_deps extern_snappy)
267+
268+
include(external/leveldb)
269+
list(APPEND third_party_deps extern_leveldb)
270+
include(external/brpc)
271+
list(APPEND third_party_deps extern_brpc)
264272
endif()
265273
endif(WITH_PSLIB)
266274

‎paddle/fluid/framework/CMakeLists.txt‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,16 @@ if (WITH_GPU)
100100
endif()
101101
cc_test(var_type_traits_test SRCS var_type_traits_test.cc DEPS var_type_traits)
102102

103+
set(BRPC_DEPS "")
104+
if(WITH_PSLIB OR WITH_PSCORE)
105+
set(BRPC_DEPS brpc)
106+
if(WITH_PSLIB_BRPC)
107+
set(BRPC_DEPS pslib_brpc)
108+
endif()
109+
endif()
110+
103111
cc_library(scope SRCS scope.cc DEPS glog threadpool xxhash var_type_traits)
104-
cc_library(device_worker SRCS device_worker.cc DEPS trainer_desc_proto lod_tensor scope)
112+
cc_library(device_worker SRCS device_worker.cc DEPS trainer_desc_proto lod_tensor scope ${BRPC_DEPS})
105113
cc_test(device_worker_test SRCS device_worker_test.cc DEPS device_worker)
106114

107115
cc_library(scope_pool SRCS scope_pool.cc DEPS scope)
@@ -243,9 +251,16 @@ if(WITH_DISTRIBUTE)
243251
fleet_wrapper heter_wrapper ps_gpu_wrapper box_wrapper lodtensor_printer
244252
lod_rank_table feed_fetch_method collective_helper ${GLOB_DISTRIBUTE_DEPS}
245253
graph_to_program_pass variable_helper data_feed_proto timer monitor
246-
heter_service_proto pslib_brpc)
254+
heter_service_proto ${BRPC_DEP})
247255
set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
256+
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
257+
set(DISTRIBUTE_COMPILE_FLAGS
258+
"${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
259+
endif()
248260
set_source_files_properties(executor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
261+
set_source_files_properties(device_worker.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
262+
set_source_files_properties(hetercpu_worker.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
263+
set_source_files_properties(heterxpu_trainer.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
249264
elseif(WITH_PSCORE)
250265
cc_library(executor SRCS executor.cc multi_trainer.cc pipeline_trainer.cc dataset_factory.cc
251266
dist_multi_trainer.cc trainer_factory.cc trainer.cc data_feed_factory.cc
@@ -280,7 +295,7 @@ elseif(WITH_PSLIB)
280295
pull_dense_worker.cc section_worker.cc device_worker_factory.cc data_set.cc DEPS op_registry
281296
device_context scope framework_proto data_feed_proto heter_service_proto trainer_desc_proto glog
282297
lod_rank_table fs shell fleet_wrapper heter_wrapper ps_gpu_wrapper box_wrapper lodtensor_printer feed_fetch_method
283-
graph_to_program_pass variable_helper timer monitor pslib_brpc )
298+
graph_to_program_pass variable_helper timer monitor ${BRPC_DEP})
284299
else()
285300
cc_library(executor SRCS executor.cc multi_trainer.cc pipeline_trainer.cc dataset_factory.cc
286301
dist_multi_trainer.cc trainer_factory.cc trainer.cc data_feed_factory.cc

‎paddle/fluid/framework/device_worker.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ limitations under the License. */
2929

3030
#include "paddle/fluid/framework/data_feed.h"
3131
#include "paddle/fluid/framework/executor_gc_helper.h"
32-
#include "paddle/fluid/framework/heter_service.h"
32+
#include "paddle/fluid/framework/heter_util.h"
3333
#include "paddle/fluid/framework/lod_tensor.h"
3434
#include "paddle/fluid/framework/op_registry.h"
3535
#include "paddle/fluid/framework/program_desc.h"

‎paddle/fluid/framework/executor.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,12 @@ limitations under the License. */
2020
#include <unordered_map>
2121
#include <vector>
2222

23-
#include "paddle/fluid/framework/data_set.h"
2423
#include "paddle/fluid/framework/executor_gc_helper.h"
2524
#include "paddle/fluid/framework/garbage_collector.h"
2625
#include "paddle/fluid/framework/op_info.h"
2726
#include "paddle/fluid/framework/program_desc.h"
2827
#include "paddle/fluid/framework/scope.h"
2928
#include "paddle/fluid/framework/tensor.h"
30-
#include "paddle/fluid/framework/trainer.h"
3129
#include "paddle/fluid/platform/device_context.h"
3230

3331
namespace paddle {

‎paddle/fluid/framework/executor_cache.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@
2222
#include <vector>
2323

2424
#include "paddle/fluid/framework/executor.h"
25+
#include "paddle/fluid/framework/op_proto_maker.h"
2526
#include "paddle/fluid/framework/program_desc.h"
2627
#include "paddle/fluid/platform/macros.h"
28+
#include "paddle/fluid/string/string_helper.h"
2729

2830
namespace paddle {
2931
namespace framework {

‎paddle/fluid/framework/fleet/CMakeLists.txt‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
if(WITH_PSLIB)
2-
cc_library(fleet_wrapper SRCS fleet_wrapper.cc DEPS framework_proto variable_helper scope pslib_brpc pslib)
2+
if(WITH_PSLIB_BRPC)
3+
set(BRPC_DEPS pslib_brpc)
4+
else()
5+
set(BRPC_DEPS brpc)
6+
endif(WITH_PSLIB_BRPC)
7+
cc_library(fleet_wrapper SRCS fleet_wrapper.cc DEPS framework_proto variable_helper scope ${BRPC_DEPS} pslib)
38
else()
49
cc_library(fleet_wrapper SRCS fleet_wrapper.cc DEPS framework_proto variable_helper scope)
510
endif(WITH_PSLIB)
611

712
if(WITH_HETERPS)
813
if(WITH_NCCL)
914
nv_library(ps_gpu_wrapper SRCS ps_gpu_wrapper.cu ps_gpu_wrapper.cc
10-
DEPS heter_ps)
15+
DEPS heter_ps ${BRPC_DEPS})
1116
add_subdirectory(heter_ps)
1217
elseif(WITH_RCCL)
1318
hip_library(ps_gpu_wrapper SRCS ps_gpu_wrapper.cu ps_gpu_wrapper.cc
14-
DEPS heter_ps)
19+
DEPS heter_ps ${BRPC_DEPS})
1520
add_subdirectory(heter_ps)
1621
endif(WITH_NCCL)
1722
else()
@@ -39,7 +44,17 @@ else()
3944
cc_library(gloo_wrapper SRCS gloo_wrapper.cc DEPS framework_proto variable_helper scope)
4045
endif(WITH_GLOO)
4146

42-
cc_library(heter_wrapper SRCS heter_wrapper.cc DEPS framework_proto device_context heter_service_proto)
47+
if(WITH_PSLIB)
48+
set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
49+
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
50+
set(DISTRIBUTE_COMPILE_FLAGS
51+
"${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
52+
endif()
53+
set_source_files_properties(heter_wrapper.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
54+
endif()
55+
56+
cc_library(heter_wrapper SRCS heter_wrapper.cc DEPS framework_proto
57+
device_context heter_service_proto ${BRPC_DEPS})
4358

4459
cc_test(test_fleet_cc SRCS test_fleet.cc DEPS fleet_wrapper gloo_wrapper fs shell)
4560

‎paddle/fluid/framework/fleet/fleet_wrapper.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ limitations under the License. */
2828
#include <unordered_map>
2929
#include <vector>
3030

31-
#include "paddle/fluid/framework/heter_service.h"
31+
#include "paddle/fluid/framework/heter_util.h"
3232
#include "paddle/fluid/framework/program_desc.h"
3333
#include "paddle/fluid/framework/scope.h"
3434
#include "paddle/fluid/framework/tensor.h"

‎paddle/fluid/framework/fleet/heter_ps/hashtable.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,16 @@ limitations under the License. */
1717
#include <limits>
1818
#include <memory>
1919
#include <vector>
20-
#ifdef PADDLE_WTIH_PSLIB
20+
#ifdef PADDLE_WITH_PSLIB
2121
#include "common_value.h" // NOLINT
2222
#endif
2323
#ifdef PADDLE_WITH_PSCORE
24+
#include "paddle/fluid/distributed/table/depends/large_scale_kv.h"
2425
#endif
2526
#include "thrust/pair.h"
2627
//#include "cudf/concurrent_unordered_map.cuh.h"
2728
#include "paddle/fluid/framework/fleet/heter_ps/cudf/concurrent_unordered_map.cuh.h"
2829
#ifdef PADDLE_WITH_HETERPS
29-
#include "paddle/fluid/distributed/table/depends/large_scale_kv.h"
3030
#include "paddle/fluid/platform/type_defs.h"
3131

3232
namespace paddle {

‎paddle/fluid/framework/fleet/heter_wrapper.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ limitations under the License. */
2525

2626
#ifdef PADDLE_WITH_PSLIB
2727
#include "paddle/fluid/framework/heter_service.h"
28+
#include "paddle/fluid/framework/heter_util.h"
2829
#include "paddle/fluid/framework/scope.h"
2930
#include "paddle/fluid/framework/tensor.h"
3031
#include "paddle/fluid/framework/variable_helper.h"

0 commit comments

Comments
 (0)