chore(scripts): remove manylinux2010 build enviroment

scripts/whl/manylinux2010/.dockerignore

-/output/
-/build_image.sh
-/build_wheel.sh

scripts/whl/manylinux2010/.gitignore

-/output/

scripts/whl/manylinux2010/Dockerfile

-FROM quay.io/pypa/manylinux2010_x86_64:2020-01-31-046f791
-
-ENV UID=1024 \
-    PATH=${PATH}:/usr/local/cuda/bin \
-    LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/opt/cudnn/lib64:/opt/tensorrt/lib \
-    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/opt/cudnn/lib64:/opt/tensorrt/lib \
-    CPATH=${CPATH}:/usr/local/cuda/include:/opt/cudnn/include:/opt/tensorrt/include
-
-ARG platform
-COPY setup_mirror.sh .
-RUN ./setup_mirror.sh "$platform"
-
-ADD init_image.sh /tmp
-RUN /tmp/init_image.sh && rm -f /tmp/init_image.sh
-

scripts/whl/manylinux2010/build_image.sh

-#!/bin/bash -e
-
-cd $(dirname $0)
-
-docker build -t env_manylinux2010:latest .

scripts/whl/manylinux2010/build_wheel.sh

-#!/bin/bash -e
-
-CWD=$(dirname $0)
-BASEDIR=$(readlink -f ${CWD}/../../..)
-OUTPUTDIR=$(readlink -f ${CWD}/output)
-USERID=$(id -u)
-TMPFS_ARGS="--tmpfs /tmp:exec"
-
-BUILD_WHL_CPU_ONLY=${BUILD_WHL_CPU_ONLY}
-if [[ -z ${BUILD_WHL_CPU_ONLY} ]]
-then
-    BUILD_WHL_CPU_ONLY="OFF"
-fi
-
-echo ${BASEDIR}
-pushd ${BASEDIR}/third_party >/dev/null
-    ./prepare.sh
-popd >/dev/null
-
-cd ${CWD}
-mkdir -p ${OUTPUTDIR}
-
-if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
-    if [[ -z ${CUDA_ROOT_DIR} ]]; then
-        echo "Environment variable CUDA_ROOT_DIR not set."
-        exit -1
-    fi
-    if [[ -z ${CUDNN_ROOT_DIR} ]]; then
-        echo "Environment variable CUDNN_ROOT_DIR not set."
-        exit -1
-    fi
-    if [[ -z ${TENSORRT_ROOT_DIR} ]]; then
-        echo "Environment variable TENSORRT_ROOT_DIR not set."
-        exit -1
-    fi
-
-    ## YOU SHOULD MODIFY CUDA VERSION AS BELOW WHEN UPGRADE
-    REQUIR_CUDA_VERSION="10010"
-    REQUIR_CUDNN_VERSION="7.6.3"
-    REQUIR_TENSORRT_VERSION="6.0.1.5"
-
-    CUDA_ROOT_DIR_=${CUDA_ROOT_DIR%*/}
-    CUDNN_ROOT_DIR_=${CUDNN_ROOT_DIR%*/}
-    TENSORRT_ROOT_DIR_=${TENSORRT_ROOT_DIR%*/}
-
-    CUDA_VERSION_PATH=${CUDA_ROOT_DIR_}/include/cuda.h
-    CUDNN_VERSION_PATH=${CUDNN_ROOT_DIR_}/include/cudnn.h
-    TENSORRT_VERSION_PATH=${TENSORRT_ROOT_DIR_}/include/NvInferVersion.h
-
-    if [ ! -e $CUDA_VERSION_PATH ] ; then
-        echo file $CUDA_VERSION_PATH is not exist
-        echo please check the Environment must use CUDA-10.1 NO.$REQUIR_CUDA_VERSION
-        exit -1
-    fi
-    if [ ! -e $CUDNN_VERSION_PATH ] ; then
-        echo file $CUDNN_VERSION_PATH is not exist
-        echo please check the Environment must use CUDNN-V$REQUIR_CUDNN_VERSION
-        exit -1
-    fi
-    if [ ! -e $TENSORRT_VERSION_PATH ] ; then
-        echo file $TENSORRT_VERSION_PATH is not exist
-        echo please check the Environment must use TensorRT-$REQUIR_TENSORRT_VERSION
-        exit -1
-    fi
-
-    CUDA_VERSION_CONTEXT=$(head -85 ${CUDA_VERSION_PATH})
-    CUDNN_VERSION_CONTEXT=$(head -62 ${CUDNN_VERSION_PATH})
-    TENSORRT_VERSION_CONTEXT=$(tail -12 ${TENSORRT_VERSION_PATH})
-
-    CUDA_API_VERSION=$(echo $CUDA_VERSION_CONTEXT | grep -Eo "define __CUDA_API_VERSION * +([0-9]+)")
-    CUDA_VERSION=${CUDA_API_VERSION:0-5}
-    echo CUDA_VERSION:$CUDA_VERSION
-
-    CUDNN_VERSION_MAJOR=$(echo $CUDNN_VERSION_CONTEXT | grep -Eo "define CUDNN_MAJOR * +([0-9]+)")
-    CUDNN_VERSION_MINOR=$(echo $CUDNN_VERSION_CONTEXT | grep -Eo "define CUDNN_MINOR * +([0-9]+)")
-    CUDNN_VERSION_PATCH=$(echo $CUDNN_VERSION_CONTEXT | grep -Eo "define CUDNN_PATCHLEVEL * +([0-9]+)")
-    CUDNN_VERSION=${CUDNN_VERSION_MAJOR:0-1}.${CUDNN_VERSION_MINOR:0-1}.${CUDNN_VERSION_PATCH:0-1}
-    echo CUDNN_VERSION:$CUDNN_VERSION
-
-    TENSORRT_VERSION_MAJOR=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_MAJOR * +([0-9]+)")
-    TENSORRT_VERSION_MINOR=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_MINOR * +([0-9]+)")
-    TENSORRT_VERSION_PATCH=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_PATCH * +([0-9]+)")
-    TENSORRT_VERSION_BUILD=$(echo $TENSORRT_VERSION_CONTEXT | grep -Eo "NV_TENSORRT_BUILD * +([0-9]+)")
-    TENSORRT_VERSION=${TENSORRT_VERSION_MAJOR:0-1}.${TENSORRT_VERSION_MINOR:0-1}.${TENSORRT_VERSION_PATCH:0-1}.${TENSORRT_VERSION_BUILD:0-1}
-    echo TENSORRT_VERSION:$TENSORRT_VERSION
-
-    if [ $CUDA_VERSION != $REQUIR_CUDA_VERSION ] ; then
-        echo please check the Environment must use CUDA-10.1 NO.$REQUIR_CUDA_VERSION
-        exit -1
-    fi
-
-    if [ $CUDNN_VERSION != $REQUIR_CUDNN_VERSION ] ; then
-        echo please check the Environment must use CUDNN-V$REQUIR_CUDNN_VERSION
-        exit -1
-    fi
-
-    if [ $TENSORRT_VERSION != $REQUIR_TENSORRT_VERSION ] ; then
-        echo please check the Environment must use TENSORRT-$REQUIR_TENSORRT_VERSION
-        exit -1
-    fi
-fi
-
-docker run -it --rm $TMPFS_ARGS -e UID=${USERID} -e LOCAL_VERSION=${LOCAL_VERSION} -e BUILD_WHL_CPU_ONLY=${BUILD_WHL_CPU_ONLY} -e ALL_PYTHON="${ALL_PYTHON}" -v ${CUDA_ROOT_DIR}:/usr/local/cuda -v ${CUDNN_ROOT_DIR}:/opt/cudnn -v ${TENSORRT_ROOT_DIR}:/opt/tensorrt -v ${BASEDIR}:/home/code -v ${OUTPUTDIR}:/home/output:rw env_manylinux2010:latest /home/code/scripts/whl/manylinux2010/do_build.sh

scripts/whl/manylinux2010/do_build.sh

-#!/bin/bash -e
-ALL_PYTHON=${ALL_PYTHON}
-if [[ -z ${ALL_PYTHON} ]]
-then
-    ALL_PYTHON="35m 36m 37m 38"
-fi
-
-BUILD_WHL_CPU_ONLY=${BUILD_WHL_CPU_ONLY}
-if [[ -z ${BUILD_WHL_CPU_ONLY} ]]
-then
-    BUILD_WHL_CPU_ONLY="OFF"
-fi
-
-SRC_DIR=$(readlink -f "`dirname $0`/../../../")
-BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
-if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
-    BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/
-fi
-NEW_LIB_PATH=core/lib
-
-function handle_strip() {
-    echo "now handle strip $1"
-    objcopy --only-keep-debug $1 $1.dbg
-    strip -s $1
-    objcopy --add-gnu-debuglink=$1.dbg $1
-    rm $1.dbg
-}
-
-function patch_elf_depend_lib() {
-    echo "handle common depend lib"
-    LIBS_DIR=${BUILD_DIR}/staging/megengine/core/lib
-    mkdir -p ${LIBS_DIR}
-    cp /usr/lib64/libatomic.so.1 ${LIBS_DIR}
-
-    patchelf --remove-rpath ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so
-    patchelf --force-rpath --set-rpath '$ORIGIN/lib' ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so
-    cp ${BUILD_DIR}/src/libmegengine_export.so ${LIBS_DIR}
-    patchelf --remove-rpath ${LIBS_DIR}/libmegengine_export.so
-    patchelf --force-rpath --set-rpath '$ORIGIN/.' ${LIBS_DIR}/libmegengine_export.so
-
-
-    if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
-        echo "handle cuda lib"
-        CUDA_VER=10.1
-        cp ${BUILD_DIR}/dnn/cuda-stub/libcuda.so ${LIBS_DIR}
-        cp /usr/local/cuda/lib64/libnvrtc.so.${CUDA_VER} ${LIBS_DIR}
-        cp /usr/local/cuda/lib64/libnvToolsExt.so.1 ${LIBS_DIR}
-    fi
-}
-
-for ver in ${ALL_PYTHON}
-do
-    USE_AUDITWHEEL="ON"
-    python_ver=${ver:0:2}
-    MAJOR=${python_ver:0:1}
-    MINOR=${ver:1}
-    PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver}/
-    export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
-    export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_PREFIX_PATH=${PYTHON_DIR}"
-    export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3"
-    export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}lib/"
-    export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python${MAJOR}.${MINOR}"
-    export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON"
-
-    if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
-        ${SRC_DIR}/scripts/cmake-build/host_build.sh -c -t -r
-    else
-        ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
-    fi
-
-    cd ${BUILD_DIR}
-    rm -rf staging
-    mkdir -p staging
-    cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
-
-
-    if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
-        cd ${BUILD_DIR}/dnn/cuda-stub/
-        strip -s libcuda.so
-        ln -sf libcuda.so libcuda.so.1
-    fi
-
-    handle_strip ${BUILD_DIR}/src/libmegengine_export.so
-
-    cd ${BUILD_DIR}/staging/megengine/core
-    handle_strip _imperative_rt.so
-
-    mkdir -p lib/ucx
-
-    if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
-        cp -L /usr/local/cuda/lib*/libnvrtc-builtins.so lib
-    fi
-
-
-    if [ ${USE_AUDITWHEEL} = "OFF" ]; then
-        patch_elf_depend_lib
-    fi
-
-    cd ${BUILD_DIR}/staging/
-    ${PYTHON_DIR}/bin/python setup.py bdist_wheel
-    cd /home/output
-    if [ ${USE_AUDITWHEEL} = "ON" ]; then
-        LD_LIBRARY_PATH=${BUILD_DIR}/dnn/cuda-stub:$LD_LIBRARY_PATH auditwheel repair -L ${NEW_LIB_PATH} ${BUILD_DIR}/staging/dist/Meg*.whl
-    else
-        mkdir -p ${SRC_DIR}/scripts/whl/manylinux2010/output/wheelhouse
-        cd ${BUILD_DIR}/staging/dist/
-        org_whl_name=`ls Meg*${ver}*.whl`
-        compat_whl_name=`echo ${org_whl_name} | sed 's/linux/manylinux2010/'`
-        echo "org whl name: ${org_whl_name}"
-        echo "comapt whl name: ${compat_whl_name}"
-        mv ${org_whl_name} ${SRC_DIR}/scripts/whl/manylinux2010/output/wheelhouse/${compat_whl_name}
-        cd /home/output
-    fi
-    chown -R ${UID}.${UID} .
-    # compat for root-less docker env to remove output at host side
-    chmod -R 777 .
-done

scripts/whl/manylinux2010/init_image.sh

-#!/bin/bash -e
-
-GET_PIP_URL='https://bootstrap.pypa.io/get-pip.py'
-SWIG_URL='https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz?use_mirror=autoselect'
-LLVM_URL='https://github.com/llvm-mirror/llvm/archive/release_60.tar.gz' 
-CLANG_URL='https://github.com/llvm-mirror/clang/archive/release_60.tar.gz'
-
-yum erase -y cmake cmake28
-yum install -y python34-pip pcre-devel devtoolset-8-libatomic-devel.x86_64
-
-pip3 install --no-cache-dir --only-binary :all: -U pip==19.1
-pip3 install --no-cache-dir --only-binary :all: cmake==3.16.3
-
-for ver in 35m 36m 37m 38
-do
-    python_ver=${ver:0:2}
-    curl ${GET_PIP_URL} | /opt/python/cp${python_ver}-cp${ver}/bin/python - \
-	--no-cache-dir --only-binary :all:
-    /opt/python/cp${python_ver}-cp${ver}/bin/pip install \
-	--no-cache-dir --only-binary :all: numpy==1.18.1 setuptools==46.1.3
-done
-
-pushd /home >/dev/null
-    echo "Install swig"
-    curl -sSL ${SWIG_URL} | tar xz
-    pushd swig-3.0.12 >/dev/null
-        mkdir build
-       	pushd build >/dev/null
-	    ../configure
-	    make -j$(nproc)
-	    make install
-        popd >/dev/null
-    popd >/dev/null
-    rm -rf swig-3.0.12
-    
-    echo "Install llvm"
-    curl -sSL ${LLVM_URL} | tar xz
-    pushd llvm-release_60 >/dev/null
-        mkdir build
-       	pushd build >/dev/null
-            cmake .. -DCMAKE_PREFIX_PATH=/opt/python/cp36-cp36m/ \
-		-DCMAKE_BUILD_TYPE=Release
-	    make -j$(nproc)
-	    make install
-	popd >/dev/null
-    popd >/dev/null
-    rm -rf llvm-release_60
-
-    echo "Install clang"
-    curl -sSL ${CLANG_URL} | tar xz
-    pushd clang-release_60 >/dev/null
-        mkdir build
-       	pushd build >/dev/null
-            cmake .. -DCMAKE_PREFIX_PATH=/opt/python/cp36-cp36m/ \
-                -DCMAKE_BUILD_TYPE=Release
-	    make -j$(nproc)
-	    make install
-	popd >/dev/null
-    popd >/dev/null
-    rm -rf clang-release_60 
-popd >/dev/null
-
-pushd /tmp >/dev/null
-    curl -sSL https://github.com/NixOS/patchelf/archive/0.10.tar.gz | tar xz
-    pushd /tmp/patchelf-0.10 >/dev/null
-        patch -p1 <<'EOF'
-diff --git a/src/patchelf.cc b/src/patchelf.cc
-index 0b4965a..7aae7a4 100644
---- a/src/patchelf.cc
-+++ b/src/patchelf.cc
-@@ -1074,13 +1074,6 @@ void ElfFile<ElfFileParamNames>::modifySoname(sonameMode op, const std::string &
-         return;
-     }
-
--    /* Zero out the previous SONAME */
--    unsigned int sonameSize = 0;
--    if (soname) {
--        sonameSize = strlen(soname);
--        memset(soname, 'X', sonameSize);
--    }
--
-     debug("new SONAME is '%s'\n", newSoname.c_str());
-
-     /* Grow the .dynstr section to make room for the new SONAME. */
-@@ -1264,7 +1257,6 @@ void ElfFile<ElfFileParamNames>::modifyRPath(RPathOp op,
-     unsigned int rpathSize = 0;
-     if (rpath) {
-         rpathSize = strlen(rpath);
--        memset(rpath, 'X', rpathSize);
-     }
-
-     debug("new rpath is '%s'\n", newRPath.c_str());
-
-EOF
-        ./bootstrap.sh && ./configure && make install-strip
-    popd
-    rm -rf /tmp/patchelf-0.10
-popd
-
-yum clean all

scripts/whl/manylinux2010/setup_mirror.sh

-#!/bin/bash
-
-set -e
-
-function set_tuna_yum_mirror() {
-    cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak
-    local repo=/etc/yum.repos.d/CentOS-Base.repo
-    local plugin=/etc/yum/pluginconf.d/fastestmirror.conf
-    sed -i "s/mirrorlist=/#mirrorlist=/g" $repo
-    sed -i "s/#baseurl/baseurl/g" $repo
-    sed -i "s/mirror.centos.org/mirrors.tuna.tsinghua.edu.cn/g" $repo
-    sed -i "s/http/https/g" $repo
-    sed -i "s/enabled=1/enabled=0/g" $plugin
-    yum clean all
-    # Build on brainpp unable to pull epel reo metadata so disable this
-    # https://unix.stackexchange.com/questions/148144/unable-to-pull-epel-repository-metadata
-    yum --disablerepo="epel" update nss
-    yum makecache
-}
-
-function set_epel() {
-    mv /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel.repo.backup
-    mv /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel-testing.repo.backup
-    curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-6.repo
-}
-
-function set_yum_mirror() {
-    mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
-    curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-6.repo
-    yum makecache 
-}
-
-function set_pip_mirror() {
-cat > /etc/pip.conf <<EOF
-[global]
-timeout = 180
-index-url = https://mirrors.aliyun.com/pypi/simple
-extra-index-url = 
-    http://mirrors.i.brainpp.cn/pypi/simple/
-    http://pypi.i.brainpp.cn/brain/dev/+simple
-    https://pypi.tuna.tsinghua.edu.cn/simple
-trusted-host =
-    mirrors.i.brainpp.cn
-    pypi.i.brainpp.cn
-    pypi.tuna.tsinghua.edu.cn
-    mirrors.aliyun.com
-EOF
-}
-
-
-function main() {
-    local platform=$1
-    case $platform in
-        brainpp)
-            set_epel
-            set_yum_mirror
-            set_pip_mirror
-            ;;
-        *)
-            echo "No setup required"
-            ;;
-    esac
-}
-
-main "$@"