Skip to content

Commit ce35d72

Browse files
jim19930609jim19930609
committed
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into enable_complex_tests
2 parents a38bec5 + 3085d5e commit ce35d72

File tree

473 files changed

+14639
-6622
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

473 files changed

+14639
-6622
lines changed

‎cmake/external/ascend.cmake‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ endif()
9090
if (WITH_ASCEND_CL)
9191
macro(find_ascend_toolkit_version ascend_toolkit_version_info)
9292
file(READ ${ascend_toolkit_version_info} ASCEND_TOOLKIT_VERSION_CONTENTS)
93-
string(REGEX MATCH "version=([0-9]+\.[0-9]+\.[0-9]+\.[a-z]*[0-9]*)" ASCEND_TOOLKIT_VERSION "${ASCEND_TOOLKIT_VERSION_CONTENTS}")
94-
string(REGEX REPLACE "version=([0-9]+\.[0-9]+\.[0-9]+\.[a-z]*[0-9]*)" "\\1" ASCEND_TOOLKIT_VERSION "${ASCEND_TOOLKIT_VERSION}")
95-
string(REGEX REPLACE "[a-z|\.]" "" CANN_VERSION ${ASCEND_TOOLKIT_VERSION})
93+
string(REGEX MATCH "version=([0-9]+\.[0-9]+\.(RC)?[0-9]+\.[a-z]*[0-9]*)" ASCEND_TOOLKIT_VERSION "${ASCEND_TOOLKIT_VERSION_CONTENTS}")
94+
string(REGEX REPLACE "version=([0-9]+\.[0-9]+\.(RC)?[0-9]+\.[a-z]*[0-9]*)" "\\1" ASCEND_TOOLKIT_VERSION "${ASCEND_TOOLKIT_VERSION}")
95+
string(REGEX REPLACE "[A-Z]|[a-z|\.]" "" CANN_VERSION ${ASCEND_TOOLKIT_VERSION})
9696
add_definitions("-DCANN_VERSION_CODE=${CANN_VERSION}")
9797
if(NOT ASCEND_TOOLKIT_VERSION)
9898
set(ASCEND_TOOLKIT_VERSION "???")

‎cmake/phi.cmake‎

Lines changed: 58 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@ function(kernel_library TARGET)
100100
set(xpu_srcs)
101101
set(gpudnn_srcs)
102102
set(kps_srcs)
103-
set(selected_rows_srcs)
104103
# parse and save the deps kerenl targets
105104
set(all_srcs)
106105
set(kernel_deps)
@@ -111,6 +110,12 @@ function(kernel_library TARGET)
111110

112111
cmake_parse_arguments(kernel_library "${options}" "${oneValueArgs}"
113112
"${multiValueArgs}" ${ARGN})
113+
114+
# used for cc_library selected_rows dir target
115+
set(target_suffix "")
116+
if ("${kernel_library_SUB_DIR}" STREQUAL "selected_rows_kernel")
117+
set(target_suffix "_sr")
118+
endif()
114119

115120
list(LENGTH kernel_library_SRCS kernel_library_SRCS_len)
116121
# one kernel only match one impl file in each backend
@@ -121,9 +126,6 @@ function(kernel_library TARGET)
121126
if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/cpu/${TARGET}.cc AND NOT WITH_XPU_KP)
122127
list(APPEND cpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/cpu/${TARGET}.cc)
123128
endif()
124-
if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/selected_rows/${TARGET}.cc)
125-
list(APPEND selected_rows_srcs ${CMAKE_CURRENT_SOURCE_DIR}/selected_rows/${TARGET}.cc)
126-
endif()
127129
if (WITH_GPU OR WITH_ROCM)
128130
if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu)
129131
list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu)
@@ -169,119 +171,119 @@ function(kernel_library TARGET)
169171
list(APPEND all_srcs ${xpu_srcs})
170172
list(APPEND all_srcs ${gpudnn_srcs})
171173
list(APPEND all_srcs ${kps_srcs})
174+
175+
set(all_include_kernels)
176+
set(all_kernel_name)
177+
172178
foreach(src ${all_srcs})
173179
file(READ ${src} target_content)
180+
# "kernels/xxx"(DenseTensor Kernel) can only include each other, but can't include "SUB_DIR/xxx" (such as selected_rows Kernel)
174181
string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
175-
if ("${kernel_library_SUB_DIR}" STREQUAL "")
176-
string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
177-
else()
182+
list(APPEND all_include_kernels ${include_kernels})
183+
184+
# "SUB_DIR/xxx" can include "kernels/xx" and "SUB_DIR/xxx"
185+
if (NOT "${kernel_library_SUB_DIR}" STREQUAL "")
178186
string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
187+
list(APPEND all_include_kernels ${include_kernels})
179188
endif()
180-
foreach(include_kernel ${include_kernels})
189+
190+
foreach(include_kernel ${all_include_kernels})
181191
if ("${kernel_library_SUB_DIR}" STREQUAL "")
182192
string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/" "" kernel_name ${include_kernel})
193+
string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
194+
list(APPEND all_kernel_name ${kernel_name})
183195
else()
184-
string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/" "" kernel_name ${include_kernel})
196+
# NOTE(dev): we should firstly match kernel_library_SUB_DIR.
197+
if (${include_kernel} MATCHES "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/")
198+
string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/" "" kernel_name ${include_kernel})
199+
# for selected_rows directory, add ${target_suffix}.
200+
string(REGEX REPLACE ".h\"" "${target_suffix}" kernel_name ${kernel_name})
201+
list(APPEND all_kernel_name ${kernel_name})
202+
else()
203+
string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/" "" kernel_name ${include_kernel})
204+
string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
205+
list(APPEND all_kernel_name ${kernel_name})
206+
endif()
185207
endif()
186-
string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
187-
list(APPEND kernel_deps ${kernel_name})
208+
list(APPEND kernel_deps ${all_kernel_name})
188209
endforeach()
189210
endforeach()
190211
list(REMOVE_DUPLICATES kernel_deps)
191-
list(REMOVE_ITEM kernel_deps ${TARGET})
212+
list(REMOVE_ITEM kernel_deps ${TARGET}${target_suffix})
192213

193214
list(LENGTH common_srcs common_srcs_len)
194215
list(LENGTH cpu_srcs cpu_srcs_len)
195216
list(LENGTH gpu_srcs gpu_srcs_len)
196217
list(LENGTH xpu_srcs xpu_srcs_len)
197218
list(LENGTH gpudnn_srcs gpudnn_srcs_len)
198219
list(LENGTH kps_srcs kps_srcs_len)
199-
list(LENGTH selected_rows_srcs selected_rows_srcs_len)
200220

201221
# kernel source file level
202222
# level 1: base device kernel
203223
# - cpu_srcs / gpu_srcs / xpu_srcs / gpudnn_srcs / kps_srcs
204224
# level 2: device-independent kernel
205225
# - common_srcs
206-
# level 3: Kernel implemented by reusing device-independent kernel
207-
# - selected_rows_srcs
208226
set(base_device_kernels)
209227
set(device_independent_kernel)
210-
set(high_level_kernels)
211228

212229
# 1. Base device kernel compile
213230
if (${cpu_srcs_len} GREATER 0)
214-
cc_library(${TARGET}_cpu SRCS ${cpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
215-
list(APPEND base_device_kernels ${TARGET}_cpu)
231+
cc_library(${TARGET}_cpu${target_suffix} SRCS ${cpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
232+
list(APPEND base_device_kernels ${TARGET}_cpu${target_suffix})
216233
endif()
217234
if (${gpu_srcs_len} GREATER 0)
218235
if (WITH_GPU)
219-
nv_library(${TARGET}_gpu SRCS ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
236+
nv_library(${TARGET}_gpu${target_suffix} SRCS ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
220237
elseif (WITH_ROCM)
221-
hip_library(${TARGET}_gpu SRCS ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
238+
hip_library(${TARGET}_gpu${target_suffix} SRCS ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
222239
endif()
223-
list(APPEND base_device_kernels ${TARGET}_gpu)
240+
list(APPEND base_device_kernels ${TARGET}_gpu${target_suffix})
224241
endif()
225242
if (${xpu_srcs_len} GREATER 0)
226-
cc_library(${TARGET}_xpu SRCS ${xpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
227-
list(APPEND base_device_kernels ${TARGET}_xpu)
243+
cc_library(${TARGET}_xpu${target_suffix} SRCS ${xpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
244+
list(APPEND base_device_kernels ${TARGET}_xpu${target_suffix})
228245
endif()
229246
if (${gpudnn_srcs_len} GREATER 0)
230247
if (WITH_GPU)
231-
nv_library(${TARGET}_gpudnn SRCS ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
248+
nv_library(${TARGET}_gpudnn${target_suffix} SRCS ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
232249
elseif (WITH_ROCM)
233-
hip_library(${TARGET}_gpudnn SRCS ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
250+
hip_library(${TARGET}_gpudnn${target_suffix} SRCS ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
234251
endif()
235-
list(APPEND base_device_kernels ${TARGET}_gpudnn)
252+
list(APPEND base_device_kernels ${TARGET}_gpudnn${target_suffix})
236253
endif()
237254
if (${kps_srcs_len} GREATER 0)
238255
# only when WITH_XPU_KP, the kps_srcs_len can be > 0
239-
xpu_library(${TARGET}_kps SRCS ${kps_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
240-
list(APPEND base_device_kernels ${TARGET}_kps)
256+
xpu_library(${TARGET}_kps${target_suffix} SRCS ${kps_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
257+
list(APPEND base_device_kernels ${TARGET}_kps${target_suffix})
241258
endif()
242259

243260
# 2. Device-independent kernel compile
244261
if (${common_srcs_len} GREATER 0)
245262
if (WITH_GPU)
246-
nv_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
263+
nv_library(${TARGET}_common${target_suffix} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
247264
elseif (WITH_ROCM)
248-
hip_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
265+
hip_library(${TARGET}_common${target_suffix} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
249266
elseif (WITH_XPU_KP)
250-
xpu_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
267+
xpu_library(${TARGET}_common${target_suffix} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
251268
else()
252-
cc_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
269+
cc_library(${TARGET}_common${target_suffix} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
253270
endif()
254-
list(APPEND device_independent_kernel ${TARGET}_common)
271+
list(APPEND device_independent_kernel ${TARGET}_common${target_suffix})
255272
endif()
256273

257-
# 3. Reusing kernel compile
258-
if (${selected_rows_srcs_len} GREATER 0)
259-
if (WITH_GPU)
260-
nv_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
261-
elseif (WITH_ROCM)
262-
hip_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
263-
elseif (WITH_XPU_KP)
264-
xpu_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
265-
else()
266-
cc_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
267-
endif()
268-
list(APPEND high_level_kernels ${TARGET}_sr)
269-
endif()
270274

271-
# 4. Unify target compile
275+
# 3. Unify target compile
272276
list(LENGTH base_device_kernels base_device_kernels_len)
273277
list(LENGTH device_independent_kernel device_independent_kernel_len)
274-
list(LENGTH high_level_kernels high_level_kernels_len)
275-
if (${base_device_kernels_len} GREATER 0 OR ${device_independent_kernel_len} GREATER 0 OR
276-
${high_level_kernels_len} GREATER 0)
278+
if (${base_device_kernels_len} GREATER 0 OR ${device_independent_kernel_len} GREATER 0)
277279
if (WITH_GPU)
278-
nv_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
280+
nv_library(${TARGET}${target_suffix} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
279281
elseif (WITH_ROCM)
280-
hip_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
282+
hip_library(${TARGET}${target_suffix} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
281283
elseif (WITH_XPU_KP)
282-
xpu_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
284+
xpu_library(${TARGET}${target_suffix} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
283285
else()
284-
cc_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
286+
cc_library(${TARGET}${target_suffix} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
285287
endif()
286288
else()
287289
set(target_build_flag 0)
@@ -290,10 +292,10 @@ function(kernel_library TARGET)
290292
if (${target_build_flag} EQUAL 1)
291293
if (${common_srcs_len} GREATER 0 OR ${cpu_srcs_len} GREATER 0 OR
292294
${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0 OR
293-
${gpudnn_srcs_len} GREATER 0 OR ${selected_rows_srcs_len} GREATER 0)
295+
${gpudnn_srcs_len} GREATER 0)
294296
# append target into PHI_KERNELS property
295297
get_property(phi_kernels GLOBAL PROPERTY PHI_KERNELS)
296-
set(phi_kernels ${phi_kernels} ${TARGET})
298+
set(phi_kernels ${phi_kernels} ${TARGET}${target_suffix})
297299
set_property(GLOBAL PROPERTY PHI_KERNELS ${phi_kernels})
298300
endif()
299301

@@ -318,9 +320,6 @@ function(kernel_library TARGET)
318320
if (${kps_srcs_len} GREATER 0)
319321
kernel_declare(${kps_srcs})
320322
endif()
321-
if (${selected_rows_srcs_len} GREATER 0)
322-
kernel_declare(${selected_rows_srcs})
323-
endif()
324323
endif()
325324
endfunction()
326325

‎paddle/fluid/distributed/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
add_subdirectory(collective)
22
add_subdirectory(store)
33
if(NOT WITH_PSCORE)
4+
if(WITH_HETERPS)
5+
add_subdirectory(ps)
6+
endif()
47
add_subdirectory(fleet_executor)
58
return()
69
endif()

‎paddle/fluid/distributed/ps.proto‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,13 +219,13 @@ message GraphParameter {
219219
optional string gpups_graph_sample_class = 3
220220
[ default = "CompleteGraphSampler" ];
221221
optional string gpups_graph_sample_args = 4 [ default = "" ];
222-
optional bool use_cache = 5 [ default = true ];
223-
optional float cache_ratio = 6 [ default = 0.3 ];
222+
optional bool use_cache = 5 [ default = false ];
223+
optional int32 cache_size_limit = 6 [ default = 100000 ];
224224
optional int32 cache_ttl = 7 [ default = 5 ];
225225
optional GraphFeature graph_feature = 8;
226226
optional string table_name = 9 [ default = "" ];
227227
optional string table_type = 10 [ default = "" ];
228-
optional int32 gpups_mode_shard_num = 11 [ default = 127 ];
228+
optional int32 shard_num = 11 [ default = 127 ];
229229
optional int32 gpu_num = 12 [ default = 1 ];
230230
}
231231

‎paddle/fluid/distributed/ps/service/brpc_ps_client.cc‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ int32_t BrpcPsClient::initialize() {
238238
std::thread(std::bind(&BrpcPsClient::push_dense_task_consume, this));
239239
// for debug
240240
// _print_thread =
241-
// std::thread(std::bind(&BrpcPsClient::print_queue_size_thread, this));
241+
// std::thread(std::bind(&BrpcPsClient::print_queue_size_thread, this));
242242

243243
return 0;
244244
}
@@ -1315,11 +1315,11 @@ std::future<int32_t> BrpcPsClient::push_sparse(size_t table_id,
13151315
CostTimer parse_timer("pserver_client_push_sparse_parse");
13161316
int push_sparse_async_num = _push_sparse_task_queue_map[table_id]->Size();
13171317
while (push_sparse_async_num > FLAGS_pserver_max_async_call_num) {
1318-
// LOG(INFO) << "push_sparse Waiting for async_call_num comsume, task_num:"
1319-
// << push_sparse_async_num << ", max_task_limit:" <<
1320-
// FLAGS_pserver_max_async_call_num;
1318+
// LOG(INFO) << "push_sparse Waiting for async_call_num comsume,
1319+
// task_num:"
1320+
// << push_sparse_async_num
1321+
// << ", max_task_limit:" << FLAGS_pserver_max_async_call_num;
13211322
usleep(5000); // 5ms
1322-
// push_sparse_async_num = _push_sparse_task_queue_map[table_id]->size();
13231323
push_sparse_async_num = _push_sparse_task_queue_map[table_id]->Size();
13241324
}
13251325
auto put_timer = std::make_shared<CostTimer>("client_push_sparse_put");
@@ -1381,8 +1381,7 @@ void BrpcPsClient::push_sparse_task_consume() {
13811381
::ThreadPool async_push_sparse_shard_threads(
13821382
FLAGS_pserver_sparse_merge_thread);
13831383
while (_running) {
1384-
platform::Timer timeline;
1385-
timeline.Start();
1384+
auto async_start_time_ms = butil::gettimeofday_ms();
13861385
// 所有sparseTable的pushTask 进行处理
13871386
for (auto &push_sparse_task_itr : _push_sparse_task_queue_map) {
13881387
auto table_id = push_sparse_task_itr.first;
@@ -1497,9 +1496,8 @@ void BrpcPsClient::push_sparse_task_consume() {
14971496
std::vector<std::future<int>>().swap(merge_status);
14981497
}
14991498
}
1500-
timeline.Pause();
1501-
auto wait_ms =
1502-
FLAGS_pserver_async_push_sparse_interval_ms - (timeline.ElapsedMS());
1499+
auto wait_ms = FLAGS_pserver_async_push_sparse_interval_ms -
1500+
(butil::gettimeofday_ms() - async_start_time_ms);
15031501
if (wait_ms > 0) {
15041502
usleep(wait_ms * 1000);
15051503
}
@@ -1661,9 +1659,10 @@ std::future<int32_t> BrpcPsClient::push_dense(const Region *regions,
16611659
std::make_shared<CostTimer>("pserver_client_push_dense_parse");
16621660
int push_dense_async_num = _push_dense_task_queue_map[table_id]->Size();
16631661
while (push_dense_async_num > FLAGS_pserver_max_async_call_num) {
1664-
LOG(INFO) << "push_dense Waiting for async_call_num comsume, task_num:"
1665-
<< push_dense_async_num
1666-
<< ", max_task_limit:" << FLAGS_pserver_max_async_call_num;
1662+
// LOG(INFO) << "push_dense Waiting for async_call_num comsume,
1663+
// task_num:"
1664+
// << push_dense_async_num
1665+
// << ", max_task_limit:" << FLAGS_pserver_max_async_call_num;
16671666
usleep(5000); // 5ms
16681667
push_dense_async_num = _push_dense_task_queue_map[table_id]->Size();
16691668
}
@@ -1701,8 +1700,7 @@ void BrpcPsClient::push_dense_task_consume() {
17011700
static bool scale_gradient = FLAGS_pserver_scale_gradient_by_merge;
17021701
::ThreadPool async_merge_dense_threads(10);
17031702
while (_running) {
1704-
platform::Timer timeline;
1705-
timeline.Start();
1703+
auto async_start_time_ms = butil::gettimeofday_ms();
17061704
for (auto &task_queue_itr : _push_dense_task_queue_map) {
17071705
auto &task_queue = task_queue_itr.second;
17081706
auto queue_size = task_queue->Size();
@@ -1791,9 +1789,8 @@ void BrpcPsClient::push_dense_task_consume() {
17911789
push_dense_raw_gradient(task_ptr, total_send_data, total_send_data_size,
17921790
closure);
17931791
}
1794-
timeline.Pause();
1795-
auto wait_ms =
1796-
FLAGS_pserver_async_push_dense_interval_ms - (timeline.ElapsedMS());
1792+
auto wait_ms = FLAGS_pserver_async_push_dense_interval_ms -
1793+
(butil::gettimeofday_ms() - async_start_time_ms);
17971794
if (wait_ms > 0) {
17981795
usleep(wait_ms * 1000);
17991796
}

0 commit comments

Comments
 (0)