Skip to content

[Inference] fix bug of RunWithExternalStream API #64466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ming1753 merged 3 commits into from
May 21, 2024
Merged

[Inference] fix bug of RunWithExternalStream API #64466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cddac09
[Inference] fix bug of RunWithExternalStream API
ming1753 May 20, 2024
7df9528
fix bug
ming1753 May 20, 2024
b801f63
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
ming1753 May 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 2 additions & 14 deletions paddle/fluid/framework/new_executor/pir_interpreter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1377,7 +1377,7 @@ paddle::framework::FetchList PirInterpreter::Run(

FeedInput();

if (!is_build_) {
if (!is_build_ || switch_stream) {
LOG_FIRST_N(INFO, 1) << "New Executor is Running ...";
VLOG(4) << DebugValueInfo();

Expand Down Expand Up @@ -1412,12 +1412,6 @@ paddle::framework::FetchList PirInterpreter::Run(
is_build_ = true;
is_shared_results_build_ = true;
} else {
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
if (switch_stream) {
BuildInstruction();
VLOG(4) << "Done BuildInstruction";
}
#endif
if (FLAGS_enable_pir_in_executor_trace_run || onednn_op_num_ ||
execution_config_.used_for_inference ||
((execution_config_.used_for_jit || execution_config_.used_for_cinn) &&
Expand Down Expand Up @@ -1467,7 +1461,7 @@ FetchList PirInterpreter::Run(const std::vector<std::string>& feed_names,
platform::RegisterModelLayout(ir_block_, place_);
#endif

if (!is_build_) {
if (!is_build_ || switch_stream) {
LOG_FIRST_N(INFO, 1) << "New Executor is Running ...";
VLOG(4) << DebugValueInfo();

Expand Down Expand Up @@ -1503,12 +1497,6 @@ FetchList PirInterpreter::Run(const std::vector<std::string>& feed_names,
is_build_ = true;
is_shared_results_build_ = true;
} else {
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
if (switch_stream) {
BuildInstruction();
VLOG(4) << "Done BuildInstruction";
}
#endif
if (FLAGS_enable_pir_in_executor_trace_run || onednn_op_num_ ||
execution_config_.used_for_inference ||
((execution_config_.used_for_jit || execution_config_.used_for_cinn) &&
Expand Down
9 changes: 2 additions & 7 deletions paddle/fluid/framework/new_executor/program_interpreter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ FetchList ProgramInterpreter::Run(const std::vector<std::string>& feed_names,
std::vector<paddle::framework::OpFuncNode> op_func_nodes;
Build(feed_names, &op_func_nodes, switch_stream);

if (!is_build_) {
if (!is_build_ || switch_stream) {
SetFeedVarsInplaceSkip(feed_names);
// convert vec func_list to graph
Convert(&op_func_nodes);
Expand All @@ -164,11 +164,6 @@ FetchList ProgramInterpreter::Run(const std::vector<std::string>& feed_names,
is_build_ = true;
is_shared_results_build_ = true;
} else {
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
if (switch_stream) {
Convert(&op_func_nodes);
}
#endif
RunImpl();
}

Expand Down Expand Up @@ -254,7 +249,7 @@ FetchList ProgramInterpreter::Run(
bool is_build = is_build_;
Prepare(feed_names, feed_tensors, is_build, switch_stream);

if (is_build) {
if (is_build && !switch_stream) {
RunImpl();
}

Expand Down
6 changes: 4 additions & 2 deletions paddle/fluid/inference/api/analysis_predictor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1484,6 +1484,8 @@ bool AnalysisPredictor::Run(const std::vector<paddle::Tensor> &inputs,
inference::DisplayMemoryInfo(place_, "before run");
if (private_context_) {
paddle::platform::DeviceContextPool::SetDeviceContexts(&device_contexts_);
auto &pool = paddle::experimental::DeviceContextPool::Instance();
pool.SyncDeviceContext(place_);
}
paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads());
#ifdef PADDLE_WITH_DNNL
Expand Down Expand Up @@ -2409,6 +2411,8 @@ bool AnalysisPredictor::ZeroCopyRun(bool switch_stream) {
#endif
if (private_context_) {
paddle::platform::DeviceContextPool::SetDeviceContexts(&device_contexts_);
auto &pool = paddle::experimental::DeviceContextPool::Instance();
pool.SyncDeviceContext(place_);
}
paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads());
#ifdef PADDLE_WITH_DNNL
Expand Down Expand Up @@ -2530,8 +2534,6 @@ bool AnalysisPredictor::ExpRunWithExternalStream(const gpuStream_t stream) {
UpdatePrivateDeviceContext(gpu_context, gpu_resource, place_);
return std::unique_ptr<phi::DeviceContext>(gpu_context);
}));
auto &pool = paddle::experimental::DeviceContextPool::Instance();
pool.SyncDeviceContext(place_);
switch_stream = true;
}
return ZeroCopyRun(switch_stream);
Expand Down