Merge pull request #4101 from luotao1/use_op

auto generate USE_OP() in pybind.cc

Author: luotao1

.gitignore

@@ -22,6 +22,7 @@ cmake-build-*
 
 # generated while compiling
 python/paddle/v2/framework/core.so
+paddle/pybind/pybind.h
 CMakeFiles
 cmake_install.cmake
 paddle/.timestamp

cmake/cpplint.cmake

@@ -26,9 +26,9 @@ set(IGNORE_PATTERN
     .*ImportanceSampler.*
     .*cblas\\.h.*
     .*\\.pb\\.txt
-    .*LtrDataProvider.*
     .*MultiDataProvider.*
-    .*pb.*)
+    .*pb.*
+    .*pybind.h)
 
 # add_style_check_target
 #

doc/howto/dev/new_op_cn.md

@@ -34,7 +34,7 @@ Kernel实现       | CPU、GPU共享Kernel实现在`.h`文件中，否则，CPU
 注册Op           | Op注册实现在`.cc`文件；Kernel注册CPU实现在`.cc`文件中，GPU实现在`.cu`文件中
 
 
-实现新的op都添加至目录[paddle/operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators)下，文件命名以`*_op.h`（如有） 、 `*_op.cc` 、`*_op.cu`（如有）结尾。
+实现新的op都添加至目录[paddle/operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators)下，文件命名以`*_op.h`（如有） 、 `*_op.cc` 、`*_op.cu`（如有）结尾。**系统会根据文件名自动构建op和其对应的Python扩展。**
 
 
 下面以矩阵乘操作，即[MulOp](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/mul_op.cc)为例来介绍如何写带Kernel的Operator。
@@ -224,45 +224,15 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs,
 
 ### 5. 编译
 
-- 简单**无特殊依赖**的OP无需修改CMakeList.txt文件。[paddle/operators/CMakeLists.txt](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/CMakeLists.txt) 会自动将 `paddle/operators` 目录下新增的 `*_op.cc` 文件加入编译。
-- 较为复杂、**有额外依赖** 的operator仍需要修改[paddle/operators/CMakeLists.txt](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/CMakeLists.txt)。如，`mul_op` 依赖 `math_function`，需要在`CMakeLists.txt`中添加如下内容：
+运行下面命令可以进行编译：
 
-    ```
-    op_library(mul_op SRCS mul_op.cc mul_op.cu DEPS math_function)		 +
-    ```
-
-- 运行下面命令可以进行编译：
-
-    ```
-    make mul_op
-    ```
+```
+make mul_op
+```
 
 ## 绑定Python
 
-- 绑定Python
-
-    在 [`paddle/pybind/pybind.cc
-`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/pybind/pybind.cc) 使用`USE_OP`告知编译器需要链接的Op，具体解释参考[代码注释](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/op_registry.h#L81)。
-
-    ```
-    USE_OP(mul);
-    ```
-    如果只实现了CPU版本，则使用`USE_CPU_ONLY_OP`:
-
-    ```
-    USE_CPU_ONLY_OP(gather);
-    ```
-
-    如果OP不带Kernel，则使用`USE_NO_KENREL_OP`:
-
-    ```
-    USE_NO_KENREL_OP(recurrent);
-    ```
-
-
- - 生成库
-
-   `paddle/operators` 目录下新增的 `*_op.cc` 文件会被自动添加链接到生成的lib库中。
+系统会对新增的op自动绑定Python，并链接到生成的lib库中。
 
 ## 实现单元测试
 
@@ -367,3 +337,10 @@ make test ARGS="-R test_mul_op -V"
 ```bash
 ctest -R test_mul_op
 ```
+
+## 注意事项
+
+- 为每个Op创建单独的`*_op.h`（如有）、`*_op.cc`和`*_op.cu`（如有）。不允许一个文件中包含多个Op，这将会导致编译出错。
+- 注册Op时的类型名，需要和该Op的名字一样。即不允许在`A_op.cc`里面，注册`REGISTER_OP(B, ...)`等，这将会导致单元测试出错。
+- 如果Op没有实现GPU Kernel，请不要创建空的`*_op.cu`，这将会导致单元测试出错。
+- 如果多个Op依赖一些共用的函数，可以创建非`*_op.*`格式的文件来存放，如`gather.h`文件。

paddle/operators/CMakeLists.txt

@@ -1,5 +1,7 @@
 file(GLOB GENERAL_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*_op.cc")
 string(REPLACE ".cc" "" GENERAL_OPS "${GENERAL_OPS}")
+set(pybind_file ${PADDLE_SOURCE_DIR}/paddle/pybind/pybind.h)
+file(WRITE ${pybind_file} "// Generated by the paddle/operator/CMakeLists.txt.  DO NOT EDIT!\n\n")
 function(op_library TARGET)
     # op_library is a function to create op library. The interface is same as
     # cc_library. But it handle split GPU/CPU code and link some common library
@@ -11,6 +13,7 @@ function(op_library TARGET)
     set(options "")
     set(oneValueArgs "")
     set(multiValueArgs SRCS DEPS)
+    set(pybind_flag 0)
     cmake_parse_arguments(op_library "${options}" "${oneValueArgs}"
             "${multiValueArgs}" ${ARGN})
 
@@ -46,15 +49,40 @@ function(op_library TARGET)
         cc_library(${TARGET} SRCS ${cc_srcs} DEPS ${op_library_DEPS}
                 ${op_common_deps})
     endif()
+
+    # net_op doesn't need pybind
+    if ("${TARGET}" STREQUAL "net_op")
+        set(pybind_flag 1)
+    endif()
+
+    # pybind USE_NO_KERNEL_OP
+    file(READ ${TARGET}.cc TARGET_CONTENT)
+    string(REGEX MATCH "OperatorWithKernel" regex_result "${TARGET_CONTENT}")
+    string(REPLACE "_op" "" TARGET "${TARGET}")
+    if (${pybind_flag} EQUAL 0 AND regex_result STREQUAL "")
+        file(APPEND ${pybind_file} "USE_NO_KERNEL_OP(${TARGET});\n")
+        set(pybind_flag 1)
+    endif()
+
+    # pybind USE_CPU_ONLY_OP
+    list(LENGTH cu_srcs cu_srcs_len)
+    if (${pybind_flag} EQUAL 0 AND ${cu_srcs_len} EQUAL 0)
+        file(APPEND ${pybind_file} "USE_CPU_ONLY_OP(${TARGET});\n")
+        set(pybind_flag 1)
+    endif()
+
+    # pybind USE_OP
+    if (${pybind_flag} EQUAL 0)
+        file(APPEND ${pybind_file} "USE_OP(${TARGET});\n")
+    endif()
 endfunction()
 
 add_subdirectory(math)
 
 set(DEPS_OPS
     recurrent_op)
-
 op_library(recurrent_op SRCS recurrent_op.cc rnn/recurrent_op_utils.cc
-  DEPS framework_proto tensor operator net_op)
+  DEPS framework_proto tensor net_op)
 
 list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS})
 foreach(src ${GENERAL_OPS})

paddle/operators/concat_op.cu

@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/operators/cross_entropy_op.h"
+#include "paddle/operators/onehot_cross_entropy_op.h"
 
 namespace paddle {
 namespace operators {

paddle/operators/cross_entropy_op.cc …dle/operators/onehot_cross_entropy_op.ccpaddle/operators/cross_entropy_op.cc renamed to paddle/operators/onehot_cross_entropy_op.cc paddle/operators/cross_entropy_op.cc renamed to paddle/operators/onehot_cross_entropy_op.cc

@@ -23,6 +23,7 @@ limitations under the License. */
 #include "paddle/operators/recurrent_op.h"
 #include "paddle/platform/enforce.h"
 #include "paddle/platform/place.h"
+#include "paddle/pybind/pybind.h"
 #include "paddle/pybind/tensor_py.h"
 #include "paddle/string/to_string.h"
 #include "pybind11/numpy.h"
@@ -31,34 +32,6 @@ limitations under the License. */
 
 namespace py = pybind11;
 
-USE_OP(add);
-USE_OP(onehot_cross_entropy);
-USE_OP(sgd);
-USE_OP(mul);
-USE_OP(elementwise_mul);
-USE_OP(mean);
-USE_OP(sigmoid);
-USE_OP(softmax);
-USE_OP(rowwise_add);
-USE_OP(fill_zeros_like);
-USE_NO_KERNEL_OP(recurrent);
-USE_OP(gaussian_random);
-USE_OP(uniform_random);
-USE_OP(lookup_table);
-USE_OP(scale);
-USE_NO_KERNEL_OP(identity);
-USE_OP(minus);
-USE_OP(cos_sim);
-USE_CPU_ONLY_OP(gather);
-USE_OP(pad);
-USE_CPU_ONLY_OP(scatter);
-USE_OP(accuracy);
-USE_CPU_ONLY_OP(concat);
-USE_OP(top_k);
-USE_OP(squared_l2_distance);
-USE_OP(sum);
-USE_OP(reshape);
-
 namespace paddle {
 namespace framework {