Merge master to develop (part 2) (#1656)

* gpu (#1310)

* Fix snapshot (#1320)

* fix bug of snapshot

* refine distribute.sh

* use more accurate function calls

* rename function

* update for model parallel

* refine code

* feat: enhance cmake download & options (#1281)

* feat: enhance cmake download & options

* feat(tools/): add share libs build scripts

* fix: add cmake options

* feat: add 3rd party download

* chore: updat README

* fix: fix protobuf & cmake repo

* fix: fix options name

* chore: merge 3rd_party.cmake & third_party.cmake

* chore: revert pre cmake URL fix

* chore: update ExternalProject check

* fix: fix typo & missing download

* fix: fix download url

* chore: update readme

* chore: fix typo

* fix: fix bugs

* fix: fix bugs

* fix: fix pre

* print all third party libs

* refine readme

* DOWNLOAD_THIRD_PARTY -> PRECOMPILED_THIRD_PARTY

* refine readme

* minor typo fix

* Fix bug in model parallel (#1345)

* fix conv in model parallel

* add TODO

* Fix bug in gcc54 (#1352)

* fix bug in gcc 5.4

* update

* refine ibverbs lib (#1391)

* refine link ibverbs lib

* modify minor

* fix a little bug in accuracy print (#1403)

* batch num for prediction (#1405)

* batch num for prediction

* !Train() => Predict()

* fix normlization epsilon check (#1433)

* Fix normlization epsilon check (#1441)

* fix normlization epsilon check

* remove check, fix eplison value in op_conf

* align with tensorflow (#1461)

* Dev crop with random size (#1468)

* random size crop proto

* ImagePreprocessImpl::<kCropWithRandomSize>

* clang format

* MaxVal

* Dev jinyi offline build (#1476)

* chore: remove pre compiler funcs

* chore: add submoudles

* fix: fix project build URL from git_url -> submodule_dir_url

* fix: fix submodule commit id

* fix: fix .gitmodules

* chore: mv third_party dir

* chore: remove test-driver(glog#188) link in glog submodule

* fix: update glog from: da816ea70645e463aa04f9564544939fa327d5a7 ==> to: 4f3e18bf26cdb794fc66cec348f57b5838a0c929

* chore: update README.md

.gitmodules

+[submodule "cub"]
+	path = build/third_party/cub/src/cub
+	url = https://github.com/NVlabs/cub.git
+[submodule "eigen"]
+	path = build/third_party/eigen/src/eigen
+	url = https://github.com/eigenteam/eigen-git-mirror
+[submodule "gflags"]
+	path = build/third_party/gflags/src/gflags
+	url = https://github.com/gflags/gflags
+[submodule "glog"]
+	path = build/third_party/glog/src/glog
+	url = https://github.com/google/glog
+[submodule "googletest"]
+	path = build/third_party/googletest/src/googletest
+	url = https://github.com/google/googletest
+[submodule "grpc"]
+	path = build/third_party/grpc/src/grpc
+	url = https://github.com/yuanms2/grpc
+[submodule "libjpeg-turbo"]
+	path = build/third_party/libjpeg-turbo/src/libjpeg-turbo
+	url = https://github.com/libjpeg-turbo/libjpeg-turbo
+[submodule "nccl"]
+	path = build/third_party/nccl/src/nccl
+	url = https://github.com/NVIDIA/nccl
+[submodule "opencv"]
+	path = build/third_party/opencv/src/opencv
+	url = https://github.com/opencv/opencv
+[submodule "protobuf"]
+	path = build/third_party/protobuf/src/protobuf
+	url = https://github.com/protocolbuffers/protobuf
+[submodule "zlib"]
+	path = build/third_party/zlib/src/zlib
+	url = https://github.com/madler/zlib

CMakeLists.txt

 # Minimum CMake required
 cmake_minimum_required(VERSION 3.5)
 
-option(BUILD_THIRD_PARTY "Build third party or oneflow" OFF)
-option(BUILD_RDMA "" ON)
+option(THIRD_PARTY "Build third party or oneflow" OFF)
+
+option(USE_CLANG_FORMAT "" OFF)
+option(BUILD_RDMA "" OFF)
 option(BUILD_CUDA "" ON)
-option(RELEASE_VERSION "" OFF)
+option(RELEASE_VERSION "" ON)
 
 # Project
-if (BUILD_THIRD_PARTY)
+if (THIRD_PARTY)
   project(third_party C CXX)
 else()
   project(oneflow C CXX)
@@ -52,27 +54,6 @@ else()
   endif()
 endif()
 
-if (BUILD_RDMA)
-  if (UNIX)
-    include(CheckIncludeFiles)
-    include(CheckLibraryExists)
-    CHECK_INCLUDE_FILES(infiniband/verbs.h HAVE_VERBS_H)
-    CHECK_LIBRARY_EXISTS(ibverbs ibv_create_qp "" HAVE_IBVERBS)
-    if (HAVE_VERBS_H AND HAVE_IBVERBS)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -libverbs")
-      add_definitions(-DWITH_RDMA)
-    elseif(HAVE_VERBS_H)
-      message(FATAL_ERROR "RDMA library not found")
-    elseif(HAVE_IBVERBS)
-      message(FATAL_ERROR "RDMA head file not found")
-    else()
-      message(FATAL_ERROR "RDMA library and head file not found")
-    endif()
-  else()
-    message(FATAL_ERROR "UNIMPLEMENTED")
-  endif()
-endif()
-
 set(THIRD_PARTY_DIR "${PROJECT_SOURCE_DIR}/third_party"
   CACHE PATH "Where the third party headers and libs are put")
 
@@ -83,9 +64,9 @@ include(util)
 include(proto2cpp)
 include(third_party)
 
-if (BUILD_THIRD_PARTY)
+if (THIRD_PARTY)
   add_custom_target(prepare_oneflow_third_party ALL DEPENDS ${oneflow_third_party_dependencies})
-  set(BUILD_THIRD_PARTY OFF CACHE BOOL "" FORCE)
+  set(THIRD_PARTY OFF CACHE BOOL "" FORCE)
 else()
   include(oneflow)
 endif()

README.md

@@ -2,9 +2,43 @@
 
 ### 1.1 Linux 
 
+### Build
+
+Building OneFlow from source requires a `BLAS libary` installed. On CentOS, if you have `Intel MKL` installed, please update the environment variable. 
+
+```
+    export LD_LIBRARY_PATH=/opt/intel/lib/intel64_lin:/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH
+```
+
+Or you can install OpenBLAS and other tools through:
+
+```
+    sudo yum -y install epel-release && sudo yum -y install git gcc-c++ cmake3 openblas-devel kernel-devel-$(uname -r) nasm
+```
+
+#### clone source code
+
+> note: with `--recursive` flag to clone third_party submodules
+
+```
+    git clone https://github.com/Oneflow-Inc/oneflow --recursive
+```
+
+or you can just clone source code and submodules step by step
+
+```
+    git clone https://github.com/Oneflow-Inc/oneflow
+    git submodule update --init --recursive
+```
+
+#### build third party from source
+
+```
+  cmake -DTHIRD_PARTY=ON .. && make -j
+```
+
+#### build oneflow
+
 ```
-export LD_LIBRARY_PATH=/opt/intel/lib/intel64_lin:/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH
-mkdir build && cd build
-cmake -DBUILD_THIRD_PARTY=ON .. && make
-cmake -DBUILD_THIRD_PARTY=OFF .. && make
+    cmake -DTHIRD_PARTY=OFF .. && make -j
 ```

cub @ c3cceac1

+Subproject commit c3cceac115c072fb63df1836ff46d8c60d9eb304

eigen @ e9e95489

+Subproject commit e9e95489a0b241412e31f0525e85b2fab386c786

gflags @ 9314597d

+Subproject commit 9314597d4b742ed6f95665241345e590a0f5759b

glog @ 4f3e18bf

+Subproject commit 4f3e18bf26cdb794fc66cec348f57b5838a0c929

googletest @ ec44c6c1

+Subproject commit ec44c6c1675c25b9827aacd08c02433cccde7780

grpc @ e0db46e1

+Subproject commit e0db46e140405f0f94f03c9a55b302e39a514c48

libjpeg-turbo @ 3041cf67

+Subproject commit 3041cf67ffdc7230e377802cba0e5c325d6d01c6

nccl @ f93fe9bf

+Subproject commit f93fe9bfd94884cec2ba711897222e0df5569a53

opencv @ 51cef265

+Subproject commit 51cef2651e91003e6a6760f496719dbb325cfc61

protobuf @ 1d2c7b6c

+Subproject commit 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05

zlib @ 50893291

+Subproject commit 50893291621658f355bc5b4d450a8d06a563053d

cmake/oneflow.cmake

@@ -126,7 +126,9 @@ include_directories(${PROJECT_BINARY_DIR})
 oneflow_add_library(of_ccobj ${of_all_obj_cc})
 target_link_libraries(of_ccobj ${oneflow_third_party_libs})
 add_dependencies(of_ccobj of_protoobj)
-add_dependencies(of_ccobj of_format)
+if (USE_CLANG_FORMAT)
+  add_dependencies(of_ccobj of_format)
+endif()
 
 if(APPLE)
   set(of_libs -Wl,-force_load of_ccobj of_protoobj)

cmake/third_party.cmake

@@ -62,8 +62,6 @@ set(oneflow_third_party_libs
     ${COCOAPI_STATIC_LIBRARIES}
 )
 
-message(STATUS "oneflow_third_party_libs: " ${oneflow_third_party_libs})
-
 if(WIN32)
   # static gflags lib requires "PathMatchSpecA" defined in "ShLwApi.Lib"
   list(APPEND oneflow_third_party_libs "ShLwApi.Lib")
@@ -73,6 +71,9 @@ endif()
 set(oneflow_third_party_dependencies
   zlib_copy_headers_to_destination
   zlib_copy_libs_to_destination
+  protobuf_copy_headers_to_destination
+  protobuf_copy_libs_to_destination
+  protobuf_copy_binary_to_destination
   gflags_copy_headers_to_destination
   gflags_copy_libs_to_destination
   glog_copy_headers_to_destination
@@ -81,9 +82,6 @@ set(oneflow_third_party_dependencies
   googletest_copy_libs_to_destination
   googlemock_copy_headers_to_destination
   googlemock_copy_libs_to_destination
-  protobuf_copy_headers_to_destination
-  protobuf_copy_libs_to_destination
-  protobuf_copy_binary_to_destination
   grpc_copy_headers_to_destination
   grpc_copy_libs_to_destination
   opencv_copy_headers_to_destination
@@ -128,4 +126,27 @@ if (BUILD_CUDA)
 )
 endif()
 
+if(BUILD_RDMA)
+  if(UNIX)
+    include(CheckIncludeFiles)
+    include(CheckLibraryExists)
+    CHECK_INCLUDE_FILES(infiniband/verbs.h HAVE_VERBS_H)
+    CHECK_LIBRARY_EXISTS(ibverbs ibv_create_qp "" HAVE_IBVERBS)
+    if(HAVE_VERBS_H AND HAVE_IBVERBS)
+      list(APPEND oneflow_third_party_libs -libverbs)
+      add_definitions(-DWITH_RDMA)
+    elseif(HAVE_VERBS_H)
+      message(FATAL_ERROR "RDMA library not found")
+    elseif(HAVE_IBVERBS)
+      message(FATAL_ERROR "RDMA head file not found")
+    else()
+      message(FATAL_ERROR "RDMA library and head file not found")
+    endif()
+  else()
+    message(FATAL_ERROR "UNIMPLEMENTED")
+  endif()
+endif()
+
+message(STATUS "oneflow_third_party_libs: " ${oneflow_third_party_libs})
+
 add_definitions(-DHALF_ENABLE_CPP11_USER_LITERALS=0)

cmake/third_party/cocoapi.cmake

@@ -15,7 +15,7 @@ set(COCOAPI_HEADERS
     "${COCOAPI_BASE_DIR}/common/maskApi.h"
 )
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(cocoapi
     PREFIX cocoapi
@@ -47,4 +47,4 @@ add_custom_target(cocoapi_create_library_dir
 add_custom_target(cocoapi_copy_libs_to_destination
     COMMAND ${CMAKE_COMMAND} -E copy_if_different ${COCOAPI_BUILD_STATIC_LIBRARIES} ${COCOAPI_LIBRARY_DIR}
     DEPENDS cocoapi_create_library_dir)
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/cub.cmake

@@ -3,15 +3,13 @@ include (ExternalProject)
 set(CUB_INCLUDE_DIR ${THIRD_PARTY_DIR}/cub/include)
 set(CUB_BUILD_INCLUDE ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub/cub)
 
-set(CUB_URL https://github.com/NVlabs/cub.git)
-set(CUB_TAG c3cceac115c072fb63df1836ff46d8c60d9eb304)
+set(CUB_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/cub/src/cub)
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(cub
     PREFIX cub
-    GIT_REPOSITORY ${CUB_URL}
-    GIT_TAG ${CUB_TAG}
+    URL ${CUB_URL}
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
     INSTALL_COMMAND ""
@@ -24,4 +22,4 @@ add_custom_target(cub_copy_headers_to_destination
     COMMAND ${CMAKE_COMMAND} -E copy_directory ${CUB_BUILD_INCLUDE} ${CUB_INCLUDE_DIR}/cub
     DEPENDS cub_create_header_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/eigen.cmake

@@ -3,17 +3,15 @@ include (ExternalProject)
 set(EIGEN_INCLUDE_DIR ${THIRD_PARTY_DIR}/eigen/include/eigen3)
 set(EIGEN_INSTALL_DIR ${THIRD_PARTY_DIR}/eigen)
 
-set(eigen_URL https://github.com/eigenteam/eigen-git-mirror)
-set(eigen_TAG e9e95489a0b241412e31f0525e85b2fab386c786)
+set(EIGEN_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/eigen/src/eigen)
 
 add_definitions(-DEIGEN_NO_AUTOMATIC_RESIZING -DEIGEN_NO_MALLOC -DEIGEN_USE_GPU)
 
-if (BUILD_THIRD_PARTY)
+if (THIRD_PARTY)
   
 ExternalProject_Add(eigen
     PREFIX eigen
-    GIT_REPOSITORY ${eigen_URL}
-    GIT_TAG ${eigen_TAG}
+    URL ${EIGEN_URL}
     UPDATE_COMMAND ""
     INSTALL_DIR "${EIGEN_INSTALL_DIR}"
     CMAKE_CACHE_ARGS
@@ -26,4 +24,4 @@ ExternalProject_Add(eigen
 )
 
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/gflags.cmake

@@ -5,8 +5,7 @@ set(GFLAGS_LIBRARY_DIR ${THIRD_PARTY_DIR}/gflags/lib)
 
 set(gflags_HEADERS_DIR ${CMAKE_CURRENT_BINARY_DIR}/gflags/src/gflags/include)
 set(gflags_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/gflags/src/gflags/lib)
-set(gflags_URL https://github.com/gflags/gflags.git)
-set(gflags_TAG 9314597d4b742ed6f95665241345e590a0f5759b)
+set(gflags_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/src/gflags)
 
 if(WIN32)
     set(GFLAGS_BUILD_LIBRARY_DIR ${gflags_LIB_DIR}/${CMAKE_BUILD_TYPE})
@@ -24,12 +23,11 @@ foreach(LIBRARY_NAME ${GFLAGS_LIBRARY_NAMES})
     list(APPEND GFLAGS_BUILD_STATIC_LIBRARIES ${GFLAGS_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
 endforeach()
 
-if (BUILD_THIRD_PARTY)
+if (THIRD_PARTY)
   
 ExternalProject_Add(gflags
     PREFIX gflags
-    GIT_REPOSITORY ${gflags_URL}
-    GIT_TAG ${gflags_TAG}
+    URL ${gflags_URL}
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
     INSTALL_COMMAND ""
@@ -57,4 +55,4 @@ add_custom_target(gflags_copy_libs_to_destination
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GFLAGS_BUILD_STATIC_LIBRARIES} ${GFLAGS_LIBRARY_DIR}
   DEPENDS gflags_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/glog.cmake

@@ -3,8 +3,7 @@ include (ExternalProject)
 set(GLOG_INCLUDE_DIR ${THIRD_PARTY_DIR}/glog/include)
 set(GLOG_LIBRARY_DIR ${THIRD_PARTY_DIR}/glog/lib)
 
-set(glog_URL https://github.com/google/glog.git)
-set(glog_TAG da816ea70645e463aa04f9564544939fa327d5a7)
+set(glog_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/glog/src/glog)
 
 if(WIN32)
     set(GLOG_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/${CMAKE_BUILD_TYPE})
@@ -31,13 +30,12 @@ set (GLOG_PUBLIC_H
   ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/src/glog/log_severity.h
 )
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(glog
     DEPENDS gflags_copy_headers_to_destination gflags_copy_libs_to_destination
     PREFIX glog
-    GIT_REPOSITORY ${glog_URL}
-    GIT_TAG ${glog_TAG}
+    URL ${glog_URL}
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
     INSTALL_COMMAND ""
@@ -74,4 +72,4 @@ add_custom_target(glog_copy_libs_to_destination
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GLOG_BUILD_STATIC_LIBRARIES} ${GLOG_LIBRARY_DIR}
   DEPENDS glog_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/googletest.cmake

@@ -9,8 +9,7 @@ set(GOOGLEMOCK_LIBRARY_DIR ${THIRD_PARTY_DIR}/googlemock/lib)
 set(googletest_SRC_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googletest/include)
 set(googlemock_SRC_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googlemock/include)
 
-set(googletest_URL https://github.com/google/googletest.git)
-set(googletest_TAG ec44c6c1675c25b9827aacd08c02433cccde7780)
+set(googletest_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/googletest/src/googletest)
 
 if(WIN32)
     set(GOOGLETEST_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest/src/googletest/googlemock/gtest/${CMAKE_BUILD_TYPE})
@@ -39,12 +38,11 @@ foreach(LIBRARY_NAME ${GOOGLEMOCK_LIBRARY_NAMES})
     list(APPEND GOOGLEMOCK_BUILD_STATIC_LIBRARIES ${GOOGLEMOCK_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
 endforeach()
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(googletest
     PREFIX googletest
-    GIT_REPOSITORY ${googletest_URL}
-    GIT_TAG ${googletest_TAG}
+    URL ${googletest_URL}
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
     INSTALL_COMMAND ""
@@ -88,4 +86,4 @@ add_custom_target(googlemock_copy_libs_to_destination
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GOOGLEMOCK_BUILD_STATIC_LIBRARIES} ${GOOGLEMOCK_LIBRARY_DIR}
   DEPENDS googlemock_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/grpc.cmake

@@ -4,8 +4,7 @@ set(GRPC_INCLUDE_DIR ${THIRD_PARTY_DIR}/grpc/include)
 set(GRPC_LIBRARY_DIR ${THIRD_PARTY_DIR}/grpc/lib)
 
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
-set(GRPC_URL https://github.com/yuanms2/grpc.git)
-set(GRPC_TAG e0db46e140405f0f94f03c9a55b302e39a514c48)
+set(GRPC_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/grpc/src/grpc)
 
 if(WIN32)
     set(GRPC_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/${CMAKE_BUILD_TYPE})
@@ -26,13 +25,12 @@ foreach(LIBRARY_NAME ${GRPC_LIBRARY_NAMES})
     list(APPEND GRPC_BUILD_STATIC_LIBRARIES ${GRPC_BUILD_LIBRARY_DIR}/${LIBRARY_NAME})
 endforeach()
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(grpc
     PREFIX grpc
     DEPENDS protobuf zlib
-    GIT_REPOSITORY ${GRPC_URL}
-    GIT_TAG ${GRPC_TAG}
+    URL ${GRPC_URL}
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
     INSTALL_COMMAND ""
@@ -64,4 +62,4 @@ add_custom_target(grpc_copy_libs_to_destination
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GRPC_BUILD_STATIC_LIBRARIES} ${GRPC_LIBRARY_DIR}
   DEPENDS grpc_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/half.cmake

@@ -9,7 +9,7 @@ set(HALF_HEADERS
     "${HALF_BASE_DIR}/include/half.hpp"
 )
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(half
     PREFIX half
@@ -32,4 +32,4 @@ foreach(header_file ${HALF_HEADERS})
     add_custom_command(TARGET half_copy_headers_to_destination PRE_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${HALF_INCLUDE_DIR})
 endforeach()
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/libjpeg-turbo.cmake

@@ -3,8 +3,7 @@ include (ExternalProject)
 set(LIBJPEG_INCLUDE_DIR ${THIRD_PARTY_DIR}/libjpeg-turbo/include)
 set(LIBJPEG_LIBRARY_DIR ${THIRD_PARTY_DIR}/libjpeg-turbo/lib)
 
-set(LIBJPEG_URL https://github.com/libjpeg-turbo/libjpeg-turbo.git)
-set(LIBJPEG_TAG 3041cf67ffdc7230e377802cba0e5c325d6d01c6)
+set(LIBJPEG_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/libjpeg-turbo/src/libjpeg-turbo)
 
 if(WIN32)
 elseif(APPLE AND ("${CMAKE_GENERATOR}" STREQUAL "Xcode"))
@@ -46,12 +45,11 @@ set(LIBJPEG_HEADERS
      "${LIBJPEG_BUILD_LIBRARY_DIR}/turbojpeg.h"
 )
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(libjpeg-turbo
     PREFIX libjpeg-turbo
-    GIT_REPOSITORY ${LIBJPEG_URL}
-    GIT_TAG ${LIBJPEG_TAG}
+    URL ${LIBJPEG_URL}
     UPDATE_COMMAND ""
     INSTALL_COMMAND ""
     BUILD_IN_SOURCE 1
@@ -83,4 +81,4 @@ add_custom_target(libjpeg_copy_libs_to_destination
     COMMAND ${CMAKE_COMMAND} -E copy_if_different ${LIBJPEG_BUILD_STATIC_LIBRARIES} ${LIBJPEG_LIBRARY_DIR}
     DEPENDS libjpeg_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/nccl.cmake

@@ -3,8 +3,7 @@ include (ExternalProject)
 set(NCCL_INCLUDE_DIR ${THIRD_PARTY_DIR}/nccl/include)
 set(NCCL_LIBRARY_DIR ${THIRD_PARTY_DIR}/nccl/lib)
 
-set(NCCL_URL https://github.com/NVIDIA/nccl.git)
-set(NCCL_TAG f93fe9bfd94884cec2ba711897222e0df5569a53)
+set(NCCL_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/nccl/src/nccl)
 set(NCCL_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/nccl/src/nccl/build)
 
 if(WIN32)
@@ -27,12 +26,11 @@ set(NCCL_HEADERS
     "${NCCL_BUILD_DIR}/include/nccl.h"
 )
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(nccl
     PREFIX nccl
-    GIT_REPOSITORY ${NCCL_URL}
-    GIT_TAG ${NCCL_TAG}
+    URL ${NCCL_URL}
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""
     BUILD_IN_SOURCE 1
@@ -62,4 +60,4 @@ add_custom_target(nccl_copy_libs_to_destination
     COMMAND ${CMAKE_COMMAND} -E copy_if_different ${NCCL_BUILD_STATIC_LIBRARIES} ${NCCL_LIBRARY_DIR}
     DEPENDS nccl_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/opencv.cmake

@@ -5,8 +5,7 @@ set(OPENCV_LIBRARY_DIR ${THIRD_PARTY_DIR}/opencv/lib)
 set(OPENCV_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/build/install)
 
 set(OPENCV_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/src)
-set(OPENCV_URL https://github.com/Oneflow-Inc/opencv.git)
-set(OPENCV_TAG 51cef2651e91003e6a6760f496719dbb325cfc61)
+set(OPENCV_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/opencv/src/opencv)
 
 if(WIN32)
 elseif(APPLE AND ("${CMAKE_GENERATOR}" STREQUAL "Xcode"))
@@ -29,13 +28,12 @@ foreach(LIBRARY_NAME ${OPENCV_3RDPARTY_LIBRARY_NAMES})
 endforeach()
 
 
-if (BUILD_THIRD_PARTY)
+if (THIRD_PARTY)
 
 ExternalProject_Add(opencv
     DEPENDS libjpeg_copy_headers_to_destination libjpeg_copy_libs_to_destination
     PREFIX opencv
-    GIT_REPOSITORY ${OPENCV_URL}
-    GIT_TAG ${OPENCV_TAG}
+    URL ${OPENCV_URL}
     UPDATE_COMMAND ""
     PATCH_COMMAND cmake -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/opencv/src/opencv/build
     BUILD_IN_SOURCE 0
@@ -124,4 +122,4 @@ add_custom_target(opencv_copy_libs_to_destination
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${OPENCV_BUILD_STATIC_LIBRARIES} ${OPENCV_LIBRARY_DIR}
   DEPENDS opencv_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
+endif(THIRD_PARTY)

cmake/third_party/protobuf.cmake

@@ -5,8 +5,7 @@ set(PROTOBUF_LIBRARY_DIR ${THIRD_PARTY_DIR}/protobuf/lib)
 set(PROTOBUF_BINARY_DIR ${THIRD_PARTY_DIR}/protobuf/bin)
 
 set(PROTOBUF_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
-set(PROTOBUF_URL https://github.com/mrry/protobuf.git)  # Includes MSVC fix.
-set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05)
+set(PROTOBUF_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/protobuf/src/protobuf)
 
 if(WIN32)
     set(PROTOBUF_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/${CMAKE_BUILD_TYPE})
@@ -31,13 +30,12 @@ endforeach()
 set(PROTOBUF_BUILD_PROTOC_EXECUTABLE ${PROTOBUF_BUILD_LIBRARY_DIR}/${PROTOC_EXECUTABLE_NAME})
 set(PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_BINARY_DIR}/${PROTOC_EXECUTABLE_NAME})
 
-if (BUILD_THIRD_PARTY)
+if (THIRD_PARTY)
 
 ExternalProject_Add(protobuf
     PREFIX protobuf
     DEPENDS zlib
-    GIT_REPOSITORY ${PROTOBUF_URL}
-    GIT_TAG ${PROTOBUF_TAG}
+    URL ${PROTOBUF_URL}
     UPDATE_COMMAND ""
     BUILD_IN_SOURCE 1
     SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf
@@ -83,4 +81,4 @@ add_custom_target(protobuf_copy_binary_to_destination
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${PROTOBUF_BUILD_PROTOC_EXECUTABLE} ${PROTOBUF_BINARY_DIR}
   DEPENDS protobuf_create_binary_dir)
 
-endif(BUILD_THIRD_PARTY)
\ No newline at end of file
+endif(THIRD_PARTY)

cmake/third_party/zlib.cmake

@@ -3,9 +3,8 @@ include (ExternalProject)
 set(ZLIB_INCLUDE_DIR ${THIRD_PARTY_DIR}/zlib/include)
 set(ZLIB_LIBRARY_DIR ${THIRD_PARTY_DIR}/zlib/lib)
 
-set(ZLIB_URL https://github.com/madler/zlib)
 set(ZLIB_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/zlib/install)
-set(ZLIB_TAG 50893291621658f355bc5b4d450a8d06a563053d)
+set(ZLIB_URL ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib/src/zlib)
 
 if(WIN32)
     set(ZLIB_BUILD_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/zlib/install/lib)
@@ -28,12 +27,11 @@ set(ZLIB_HEADERS
     "${ZLIB_INSTALL}/include/zlib.h"
 )
 
-if(BUILD_THIRD_PARTY)
+if(THIRD_PARTY)
 
 ExternalProject_Add(zlib
     PREFIX zlib
-    GIT_REPOSITORY ${ZLIB_URL}
-    GIT_TAG ${ZLIB_TAG}
+    URL ${ZLIB_URL}
     UPDATE_COMMAND ""
     INSTALL_DIR ${ZLIB_INSTALL}
     BUILD_IN_SOURCE 1
@@ -66,4 +64,4 @@ add_custom_target(zlib_copy_libs_to_destination
     COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ZLIB_BUILD_STATIC_LIBRARIES} ${ZLIB_LIBRARY_DIR}
     DEPENDS zlib_create_library_dir)
 
-endif(BUILD_THIRD_PARTY)
\ No newline at end of file
+endif(THIRD_PARTY)

oneflow/core/actor/actor_message_bus.cpp

 #include "oneflow/core/actor/actor_message_bus.h"
-#include "oneflow/core/comm_network/comm_network.h"
 #include "oneflow/core/job/id_manager.h"
 #include "oneflow/core/job/machine_context.h"
 #include "oneflow/core/thread/thread_manager.h"

oneflow/core/actor/actor_message_bus.h

 #ifndef ONEFLOW_CORE_ACTOR_ACTOR_MESSAGE_BUS_H_
 #define ONEFLOW_CORE_ACTOR_ACTOR_MESSAGE_BUS_H_
 
-#include <stdint.h>
 #include "oneflow/core/actor/actor_message.h"
 #include "oneflow/core/comm_network/comm_network.h"
 #include "oneflow/core/common/util.h"
@@ -17,7 +16,7 @@ class ActorMsgBus final {
   void SendMsgWithoutCommNet(const ActorMsg& msg);
 
  private:
-  friend Global<ActorMsgBus>;
+  friend class Global<ActorMsgBus>;
   ActorMsgBus() = default;
 };
 

oneflow/core/kernel/conv_kernel.cu

@@ -65,7 +65,7 @@ void ConvKernel<DeviceType::kGPU, T>::KernelInitWithCudnn(const ParallelContext*
       new CudnnConvDesc(GetDataType<T>::value, in_shape, this->GetCustomizedOpConf()));
 
   if (this->template GetValFromCustomizedOpConf<bool>("use_bias")) {
-    int32_t filters = this->template GetValFromCustomizedOpConf<int32_t>("filters");
+    int32_t filters = Shape(this->GetConvKernelConf().bias()).At(0);
     if ((this->OpKernelDim() == 1) || (this->OpKernelDim() == 2)) {
       if (data_format == "channels_first") {
         this->bias_desc_.reset(

oneflow/core/kernel/kernel.proto

@@ -10,14 +10,15 @@ message ConvKernelConf {
   required ShapeProto in = 1;
   required ShapeProto out = 2;
   required ShapeProto weight = 3;
-  required int32 dim = 4;
-  repeated int32 pad_small_side = 5;
-  repeated int32 pad_large_side = 6;
-  repeated int32 dilation_rate = 7;
-  repeated int32 strides = 8;
-  optional int32 cudnn_fwd_algo = 9 [default = -1];
-  optional int32 cudnn_bwd_filter_algo = 10 [default = -1];
-  optional int32 cudnn_bwd_data_algo = 11 [default = -1];
+  optional ShapeProto bias = 4;
+  required int32 dim = 5;
+  repeated int32 pad_small_side = 6;
+  repeated int32 pad_large_side = 7;
+  repeated int32 dilation_rate = 8;
+  repeated int32 strides = 9;
+  optional int32 cudnn_fwd_algo = 10 [default = -1];
+  optional int32 cudnn_bwd_filter_algo = 11 [default = -1];
+  optional int32 cudnn_bwd_data_algo = 12 [default = -1];
 }
 
 message DropoutKernelConf {

oneflow/core/operator/conv_op.cpp

@@ -244,6 +244,9 @@ void ConvOp<NDims>::GenKernelConfWithCudnn(
   GetBlobDesc4BnInOp("in")->shape().ToProto(conv_conf->mutable_in());
   GetBlobDesc4BnInOp("out")->shape().ToProto(conv_conf->mutable_out());
   GetBlobDesc4BnInOp("weight")->shape().ToProto(conv_conf->mutable_weight());
+  if (GetValFromCustomizedConf<bool>("use_bias")) {
+    GetBlobDesc4BnInOp("bias")->shape().ToProto(conv_conf->mutable_bias());
+  }
 
   std::vector<int32_t> pad_small_side;
   std::vector<int32_t> pad_large_side;

oneflow/core/operator/normalization_op.cpp

@@ -5,7 +5,10 @@ namespace oneflow {
 
 void NormalizationOp::InitFromOpConf() {
   const auto& conf = op_conf().normalization_conf();
-  CHECK_GT(conf.epsilon(), 0.f);
+  float min_epsilon = CUDNN_BN_MIN_EPSILON + 1e-8;
+  if (conf.epsilon() < min_epsilon) {
+    this->mut_op_conf()->mutable_normalization_conf()->set_epsilon(min_epsilon);
+  }
   CHECK_GE(conf.momentum(), 0);
   CHECK_LE(conf.momentum(), 1);
   EnrollInputBn("in");
@@ -141,7 +144,6 @@ void NormalizationOp::InferBlobDescsForCudnn(
   const BlobDesc* in_blob_desc = GetBlobDesc4BnInOp("in");
   const DataType in_data_type = in_blob_desc->data_type();
   CHECK(conf.scale() && conf.center()) << "Cudnn batch norm must use scale and center";
-  CHECK_GT(conf.epsilon(), CUDNN_BN_MIN_EPSILON);
   InferParamBlobDescs(GetBlobDesc4BnInOp, conf, in_blob_desc->shape().At(conf.axis()), in_data_type,
                       true);
 }

oneflow/core/record/image.proto

@@ -17,10 +17,25 @@ message ImageCrop {
   required int32 height = 5;
 }
 
+message ImageCropWithRandomSize {
+  message AspectRatioRange {
+    optional float min = 1 [default = 0.75];
+    optional float max = 2 [default = 1.33];
+  }
+  message AreaRange {
+    optional float min = 1 [default = 0.05];
+    optional float max = 2 [default = 1.0];
+  }
+  required AspectRatioRange aspect_ratio_range = 1;
+  required AreaRange area_range = 2;
+  optional int32 max_attempts = 3 [default = 100];
+}
+
 message ImagePreprocess {
   oneof preprocess {
     ImageResize resize = 1;
     ImageCrop crop = 2;
-    ImageMirror mirror = 3;
+    ImageCropWithRandomSize crop_with_random_size = 3;
+    ImageMirror mirror = 4;
   }
 }

oneflow/core/record/image_preprocess.cpp

@@ -2,6 +2,15 @@
 
 namespace oneflow {
 
+namespace {
+
+float GetRandomFloatValue(float min, float max, std::function<int32_t(void)> NextRandomInt) {
+  float ratio = static_cast<float>(NextRandomInt()) / static_cast<float>(GetMaxVal<int32_t>());
+  return (max - min) * ratio + min;
+}
+
+}  // namespace
+
 void ImagePreprocessImpl<PreprocessCase::kResize>::DoPreprocess(
     cv::Mat* image, const ImagePreprocess& preprocess_conf,
     std::function<int32_t(void)> NextRandomInt) const {
@@ -24,8 +33,10 @@ void ImagePreprocessImpl<PreprocessCase::kCrop>::DoPreprocess(
   CHECK_LE(width, image->cols);
   CHECK_LE(height, image->rows);
   if (crop.random_xy()) {
-    x = NextRandomInt() % (image->cols - width);
-    y = NextRandomInt() % (image->rows - height);
+    int32_t x_max = (image->cols - width);
+    int32_t y_max = (image->rows - height);
+    x = x_max > 0 ? (NextRandomInt() % x_max) : x_max;
+    y = y_max > 0 ? (NextRandomInt() % y_max) : y_max;
   } else {
     CHECK_LE(x, image->cols - width);
     CHECK_LE(y, image->rows - height);
@@ -33,6 +44,38 @@ void ImagePreprocessImpl<PreprocessCase::kCrop>::DoPreprocess(
   *image = (*image)(cv::Rect(x, y, width, height));
 }
 
+void ImagePreprocessImpl<PreprocessCase::kCropWithRandomSize>::DoPreprocess(
+    cv::Mat* image, const ImagePreprocess& preprocess_conf,
+    std::function<int32_t(void)> NextRandomInt) const {
+  CHECK(preprocess_conf.has_crop_with_random_size());
+  const ImageCropWithRandomSize& conf = preprocess_conf.crop_with_random_size();
+  int32_t max_attempts = conf.max_attempts();
+  float area_min = conf.area_range().min();
+  float area_max = conf.area_range().max();
+  float ratio_min = conf.aspect_ratio_range().min();
+  float ratio_max = conf.aspect_ratio_range().max();
+  CHECK_LE(area_min, area_max);
+  CHECK_GT(area_min, 0.0);
+  CHECK_LE(area_max, 1.0);
+  CHECK_LE(ratio_min, ratio_max);
+  CHECK_GT(ratio_min, 0.0);
+  while (max_attempts--) {
+    float area_size = GetRandomFloatValue(area_min, area_max, NextRandomInt) * image->total();
+    float aspect_ratio = GetRandomFloatValue(ratio_min, ratio_max, NextRandomInt);
+    float height_float = sqrt(area_size / aspect_ratio);
+    int32_t height = static_cast<int32_t>(height_float);
+    int32_t width = static_cast<int32_t>(height_float * aspect_ratio);
+    if (width <= image->cols && height <= image->rows) {
+      int32_t x_max = (image->cols - width);
+      int32_t y_max = (image->rows - height);
+      int32_t x = x_max > 0 ? (NextRandomInt() % x_max) : x_max;
+      int32_t y = y_max > 0 ? (NextRandomInt() % y_max) : y_max;
+      *image = (*image)(cv::Rect(x, y, width, height));
+      return;
+    }
+  }
+}
+
 void ImagePreprocessImpl<PreprocessCase::kMirror>::DoPreprocess(
     cv::Mat* image, const ImagePreprocess& preprocess_conf,
     std::function<int32_t(void)> NextRandomInt) const {

oneflow/core/record/image_preprocess.h

@@ -42,6 +42,14 @@ class ImagePreprocessImpl<PreprocessCase::kCrop> final : public ImagePreprocessI
                     std::function<int32_t(void)> NextRandomInt) const override;
 };
 
+template<>
+class ImagePreprocessImpl<PreprocessCase::kCropWithRandomSize> final : public ImagePreprocessIf {
+ public:
+ private:
+  void DoPreprocess(cv::Mat* image, const ImagePreprocess& preprocess_conf,
+                    std::function<int32_t(void)> NextRandomInt) const override;
+};
+
 template<>
 class ImagePreprocessImpl<PreprocessCase::kMirror> final : public ImagePreprocessIf {
  public:
@@ -53,7 +61,8 @@ class ImagePreprocessImpl<PreprocessCase::kMirror> final : public ImagePreproces
 #define PREPROCESS_CASE_SEQ                     \
   OF_PP_MAKE_TUPLE_SEQ(PreprocessCase::kResize) \
   OF_PP_MAKE_TUPLE_SEQ(PreprocessCase::kMirror) \
-  OF_PP_MAKE_TUPLE_SEQ(PreprocessCase::kCrop)
+  OF_PP_MAKE_TUPLE_SEQ(PreprocessCase::kCrop)   \
+  OF_PP_MAKE_TUPLE_SEQ(PreprocessCase::kCropWithRandomSize)
 
 ImagePreprocessIf* GetImagePreprocess(PreprocessCase);