fix(cmake): opt cmake script: support list option and config ninja max jobs

GitOrigin-RevId: ef6c4a10f55510c7246ebaafd01a8a9a6902131b

fix(cmake): opt cmake script: support list option and config ninja max jobs
GitOrigin-RevId: ef6c4a10f55510c7246ebaafd01a8a9a6902131b
26ac2b00 · Megvii Engine Team · af80d365 · 26ac2b00 · 26ac2b00 · 26ac2b00
5 changed file
--- a/scripts/cmake-build/cross_build_android_arm_inference.sh
+++ b/scripts/cmake-build/cross_build_android_arm_inference.sh
@@ -10,6 +10,16 @@ REMOVE_OLD_BUILD=false
 NINJA_VERBOSE=OFF
 NINJA_DRY_RUN=OFF
 SPECIFIED_TARGET="install/strip"
+READLINK=readlink
+OS=$(uname -s)
+
+if [ $OS = "Darwin" ];then
+    READLINK=greadlink
+fi
+
+SRC_DIR=$($READLINK -f "`dirname $0`/../../")
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
+config_ninja_default_max_jobs

 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

@@ -23,16 +33,27 @@ function usage() {
    echo "-r : remove old build dir before make, default off"
    echo "-v : ninja with verbose and explain, default off"
    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
+    echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2"
    echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)"
+    echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS"
    echo "-h : show usage"
-    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
-    echo "example: $0 -d"
+    echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:"
+    echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d"
    exit -1
 }

-while getopts "nvrkhdfa:e:" arg
+while getopts "lnvrkhdfa:e:j:" arg
 do
    case $arg in
+        j)
+            NINJA_MAX_JOBS=$OPTARG
+            echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}"
+            ;;
+        l)
+            echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS"
+            show_cmakelist_options
+            exit 0
+            ;;
        d)
            echo "Build with Debug mode"
            BUILD_TYPE=Debug
@@ -95,18 +116,10 @@ echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}"
 echo "ARCH: $ARCH"
 echo "----------------------------------------------------"

-READLINK=readlink
-OS=$(uname -s)
-
-if [ $OS = "Darwin" ];then
-    READLINK=greadlink
-elif [[ $OS =~ "NT" ]]; then
+if [[ $OS =~ "NT" ]]; then
    echo "BUILD in NT ..."
 fi

-SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-source $SRC_DIR/scripts/cmake-build/utils/utils.sh
-
 if [ -z $NDK_ROOT ];then
    echo "can not find NDK_ROOT env, pls export you NDK root dir to NDK_ROOT"
    exit -1
@@ -147,7 +160,7 @@ function cmake_build() {
        ${EXTRA_CMAKE_ARGS} \
        $SRC_DIR "

-    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN}
+    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS}
    bash -c "${NINJA_CMD}"
 }


--- a/scripts/cmake-build/cross_build_ios_arm_inference.sh
+++ b/scripts/cmake-build/cross_build_ios_arm_inference.sh
@@ -10,6 +10,16 @@ REMOVE_OLD_BUILD=false
 NINJA_VERBOSE=OFF
 NINJA_DRY_RUN=OFF
 SPECIFIED_TARGET="install"
+READLINK=readlink
+OS=$(uname -s)
+
+if [ $OS = "Darwin" ];then
+    READLINK=greadlink
+fi
+
+SRC_DIR=$($READLINK -f "`dirname $0`/../../")
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
+config_ninja_default_max_jobs

 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

@@ -23,16 +33,27 @@ function usage() {
    echo "-r : remove old build dir before make, default off"
    echo "-v : ninja with verbose and explain, default off"
    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
+    echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2"
    echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)"
+    echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS"
    echo "-h : show usage"
-    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
-    echo "example: $0 -d"
+    echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:"
+    echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d"
    exit -1
 }

-while getopts "nvrkhdfa:e:" arg
+while getopts "lnvrkhdfa:e:j:" arg
 do
    case $arg in
+        j)
+            NINJA_MAX_JOBS=$OPTARG
+            echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}"
+            ;;
+        l)
+            echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS"
+            show_cmakelist_options
+            exit 0
+            ;;
        d)
            echo "Build with Debug mode"
            BUILD_TYPE=Debug
@@ -92,22 +113,17 @@ echo "BUILD_TYPE: $BUILD_TYPE"
 echo "MGE_ARMV8_2_FEATURE_FP16: $MGE_ARMV8_2_FEATURE_FP16"
 echo "MGE_DISABLE_FLOAT16: $MGE_DISABLE_FLOAT16"
 echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}"
+echo "NINJA_MAX_JOBS: ${NINJA_MAX_JOBS}"
 echo "ARCH: $ARCH"
 echo "----------------------------------------------------"

-READLINK=readlink
-OS=$(uname -s)
-
 if [ $OS = "Darwin" ];then
-    READLINK=greadlink
+    echo "cross build ios from MacOS"
 else
-    echo "cross build ios only support on macos, abort now!!"
+    echo "cross build ios only support on MacOS, abort now!!"
    exit -1
 fi

-SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-source $SRC_DIR/scripts/cmake-build/utils/utils.sh
-
 function cmake_build() {
    BUILD_DIR=$SRC_DIR/build_dir/apple/$3/$4/$1/$BUILD_TYPE/build
    INSTALL_DIR=$BUILD_DIR/../install
@@ -145,7 +161,7 @@ function cmake_build() {
        ${EXTRA_CMAKE_ARGS} \
        $SRC_DIR "

-    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN}
+    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS}
    bash -c "${NINJA_CMD}"
 }


--- a/scripts/cmake-build/cross_build_linux_arm_inference.sh
+++ b/scripts/cmake-build/cross_build_linux_arm_inference.sh
@@ -13,6 +13,16 @@ NINJA_DRY_RUN=OFF
 SPECIFIED_TARGET="install/strip"
 CMAKE_C_FLAGS="-Wno-psabi"
 CMAKE_CXX_FLAGS="-Wno-psabi"
+READLINK=readlink
+OS=$(uname -s)
+
+if [ $OS = "Darwin" ];then
+    READLINK=greadlink
+fi
+
+SRC_DIR=$($READLINK -f "`dirname $0`/../../")
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
+config_ninja_default_max_jobs

 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

@@ -27,16 +37,27 @@ function usage() {
    echo "-r : remove old build dir before make, default off"
    echo "-v : ninja with verbose and explain, default off"
    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
+    echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2"
    echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)"
+    echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS"
    echo "-h : show usage"
-    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
-    echo "example: $0 -d"
+    echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:"
+    echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d"
    exit -1
 }

-while getopts "nvrkhdcfa:e:" arg
+while getopts "lnvrkhdcfa:e:j:" arg
 do
    case $arg in
+        j)
+            NINJA_MAX_JOBS=$OPTARG
+            echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}"
+            ;;
+        l)
+            echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS"
+            show_cmakelist_options
+            exit 0
+            ;;
        d)
            echo "Build with Debug mode"
            BUILD_TYPE=Debug
@@ -101,15 +122,11 @@ echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
 echo "MGE_ARMV8_2_FEATURE_FP16: $MGE_ARMV8_2_FEATURE_FP16"
 echo "MGE_DISABLE_FLOAT16: $MGE_DISABLE_FLOAT16"
 echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}"
+echo "NINJA_MAX_JOBS: ${NINJA_MAX_JOBS}"
 echo "ARCH: $ARCH"
 echo "----------------------------------------------------"

-READLINK=readlink
-OS=$(uname -s)
-
-if [ $OS = "Darwin" ];then
-    READLINK=greadlink
-elif [[ $OS =~ "NT" ]]; then
+if [[ $OS =~ "NT" ]]; then
    echo "BUILD in NT ..."
 fi

@@ -129,9 +146,6 @@ if [ $MGE_WITH_CUDA = "OFF" ];then
    CMAKE_CXX_FLAGS="-Werror=unused-parameter -Wno-psabi"
 fi

-SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-source $SRC_DIR/scripts/cmake-build/utils/utils.sh
-
 function cmake_build() {
    BUILD_DIR=$SRC_DIR/build_dir/gnu-linux/MGE_WITH_CUDA_$3/$1/$BUILD_TYPE/build
    INSTALL_DIR=$BUILD_DIR/../install
@@ -161,7 +175,7 @@ function cmake_build() {
        ${EXTRA_CMAKE_ARGS} \
        $SRC_DIR "

-    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN}
+    config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS}
    bash -c "${NINJA_CMD}"
 }


--- a/scripts/cmake-build/host_build.sh
+++ b/scripts/cmake-build/host_build.sh
 #!/usr/bin/env bash
 set -e

-function usage() {
-    echo "$0 args1 args2 .."
-    echo "available args detail:"
-    echo "-d : Build with Debug mode, default Release mode"
-    echo "-c : Build with CUDA, default without CUDA"
-    echo "-t : Build with training mode, default inference only"
-    echo "-m : Build with m32 mode(only for windows build), default m64"
-    echo "-r : remove old build dir before make, default off"
-    echo "-v : ninja with verbose and explain, default off"
-    echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel"
-    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
-    echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)"
-    echo "-h : show usage"
-    echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
-    echo "example: $0 -d"
-    exit -1
-}
-
-READLINK=readlink
-OS=$(uname -s)
 BUILD_TYPE=Release
 MGE_WITH_CUDA=OFF
 MGE_INFERENCE_ONLY=ON
@@ -32,16 +12,56 @@ NINJA_VERBOSE=OFF
 BUILD_DEVELOP=ON
 NINJA_DRY_RUN=OFF
 SPECIFIED_TARGET="install/strip"
+READLINK=readlink
+
+OS=$(uname -s)
 if [[ $OS =~ "NT" ]]; then
    echo "Windows no need strip, caused by pdb file always split with exe"
    SPECIFIED_TARGET="install"
 fi

+if [ $OS = "Darwin" ];then
+    READLINK=greadlink
+fi
+
+SRC_DIR=$($READLINK -f "`dirname $0`/../../")
+source $SRC_DIR/scripts/cmake-build/utils/utils.sh
+config_ninja_default_max_jobs
+
 echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"

-while getopts "nsrhdctmve:" arg
+function usage() {
+    echo "$0 args1 args2 .."
+    echo "available args detail:"
+    echo "-d : Build with Debug mode, default Release mode"
+    echo "-c : Build with CUDA, default without CUDA"
+    echo "-t : Build with training mode, default inference only"
+    echo "-m : Build with m32 mode(only for windows build), default m64"
+    echo "-r : remove old build dir before make, default off"
+    echo "-v : ninja with verbose and explain, default off"
+    echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel"
+    echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
+    echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2"
+    echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)"
+    echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS"
+    echo "-h : show usage"
+    echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:"
+    echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d"
+    exit -1
+}
+
+while getopts "lnsrhdctmve:j:" arg
 do
    case $arg in
+        j)
+            NINJA_MAX_JOBS=$OPTARG
+            echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}"
+            ;;
+        l)
+            echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS"
+            show_cmakelist_options
+            exit 0
+            ;;
        d)
            echo "Build with Debug mode"
            BUILD_TYPE=Debug
@@ -95,10 +115,10 @@ echo "BUILD_TYPE: $BUILD_TYPE"
 echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
 echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
 echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}"
+echo "NINJA_MAX_JOBS: ${NINJA_MAX_JOBS}"
 echo "------------------------------------"

 if [ $OS = "Darwin" ];then
-    READLINK=greadlink
    if [ $MGE_WITH_CUDA = "ON" ];then
        echo "MACOS DO NOT SUPPORT TensorRT, ABORT NOW!!"
        exit -1
@@ -107,9 +127,6 @@ elif [[ $OS =~ "NT" ]]; then
    echo "BUILD in NT ..."
 fi

-SRC_DIR=$($READLINK -f "`dirname $0`/../../")
-source $SRC_DIR/scripts/cmake-build/utils/utils.sh
-
 if [ ${MGE_INFERENCE_ONLY} = "ON" ]; then
    echo "config BUILD_DEVELOP=OFF when MGE_INFERENCE_ONLY=ON"
    BUILD_DEVELOP=OFF
@@ -141,7 +158,7 @@ function cmake_build() {
        ${EXTRA_CMAKE_ARGS} \
        ${SRC_DIR} "

-    config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN}
+    config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS}
    bash -c "${NINJA_CMD}"
 }

@@ -280,7 +297,7 @@ function cmake_build_windows() {
        -DCMAKE_MAKE_PROGRAM=ninja.exe \
        ${EXTRA_CMAKE_ARGS} ../../.. "

-    config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN}
+    config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS}
    cmd.exe /c " vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH && ${NINJA_CMD} "
 }


--- a/scripts/cmake-build/utils/utils.sh
+++ b/scripts/cmake-build/utils/utils.sh
@@ -4,6 +4,8 @@ set -e
 OS=$(uname -s)
 NINJA_CMD=""
 NINJA_BASE="ninja"
+cpu_number=`nproc`
+NINJA_MAX_JOBS=0

 if [[ $OS =~ "NT" ]]; then
    echo "BUILD in NT ..."
@@ -15,7 +17,7 @@ if [ $OS = "Darwin" ];then
    READLINK=greadlink
 fi

-PROJECT_DIR=$(dirname "${BASH_SOURCE[0]}")/../../../
+PROJECT_DIR=$(dirname "${BASH_SOURCE[0]}")/../../..
 function cd_real_build_dir() {
    REAL_DIR=$($READLINK -f $1)
    echo "may alias dir: $1"
@@ -77,21 +79,22 @@ function try_remove_old_build() {
 }

 function config_ninja_target_cmd() {
-    if [ $# -eq 4 ]; then
+    if [ $# -eq 5 ]; then
        _NINJA_VERBOSE=$1
        _BUILD_DEVELOP=$2
        _NINJA_TARGET=$3
        _NINJA_DRY_RUN=$4
+        _NINJA_MAX_JOBS=$5
    else
        echo "err call config_ninja_target_cmd"
        exit -1
    fi
    if [ -z "${_NINJA_TARGET}" ]; then
-        NINJA_CMD="${NINJA_BASE} all"
+        NINJA_CMD="${NINJA_BASE} all -j ${_NINJA_MAX_JOBS}"
    elif [[ ${_NINJA_TARGET} =~ "install" ]]; then
-        NINJA_CMD="${NINJA_BASE} all && ${NINJA_BASE} ${_NINJA_TARGET}"
+        NINJA_CMD="${NINJA_BASE} all -j ${_NINJA_MAX_JOBS} && ${NINJA_BASE} ${_NINJA_TARGET}"
    else
-        NINJA_CMD="${NINJA_BASE} ${_NINJA_TARGET}"
+        NINJA_CMD="${NINJA_BASE} ${_NINJA_TARGET} -j ${_NINJA_MAX_JOBS}"
    fi

    if [ ${_NINJA_DRY_RUN} = "ON" ]; then
@@ -108,3 +111,14 @@ function config_ninja_target_cmd() {

    echo "build ${NINJA_BASE} target command: ${NINJA_CMD}"
 }
+
+function show_cmakelist_options() {
+    cd ${PROJECT_DIR}
+    grep "option(" lite/CMakeLists.txt CMakeLists.txt --color
+    cd - > /dev/null
+}
+
+function config_ninja_default_max_jobs() {
+    ((NINJA_MAX_JOBS = ${cpu_number} + 2))
+    echo "config default NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}"
+}