diff --git a/CMakeLists.txt b/CMakeLists.txt index ddfb4967e86b73969de61b4fbecd52a385b31290..29e9da557588497054f52001e26c5f89c181b77d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1274,16 +1274,30 @@ if(MGE_ARCH STREQUAL "aarch64") set(MARCH "-march=armv8.2-a+fp16") endif() - if(MGE_WITH_CUDA) + if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") message( WARNING - "aarch64 ld will add -mfix-cortex-a53-843419 and -mfix-cortex-a53-835769,\ - when cuda enable and CMAKE with DEBUG build type,ld will take about 14min+,\ - for save link time(14min->1min), you may open below flags if not deploy on\ - arm a53 platform, or just build release type!") + "aarch64 ld will add -mfix-cortex-a53-843419 and -mfix-cortex-a53-835769 by default.\ + when build with DEBUG build type,ld will take about 14min+, for save link time(14min->1min), \ + you may open below flags if not deploy on arm a53 platform, or just build release type!" + ) # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-fix-cortex-a53-843419 # -mno-fix-cortex-a53-835769") endif() + + if(MGE_WITH_CUDA) + message(STATUS "check compiler version...") + if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.5) + message(FATAL_ERROR "gcc version must >= 7.5 when build with cuda") + endif() + # cuda libs build with -mno-fix-cortex-a53-843419 -mno-fix-cortex-a53-835769, if you are not deploy on arm a53 platform and want to save link time, you may open below flags even + message( + WARNING + "cuda libs build with -mno-fix-cortex-a53-843419 -mno-fix-cortex-a53-835769, so force disable it to avoid link error" + ) + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -mno-fix-cortex-a53-843419 -mno-fix-cortex-a53-835769") + endif() endif() if(MGE_ARCH STREQUAL "riscv64") diff --git a/LICENSE b/LICENSE index b0403256d5c67eec0416077f69a74aac4348d0b0..38ee9fcac19d62f2f0b16b4549f688ac6b6e2c63 100644 --- a/LICENSE +++ b/LICENSE @@ -175,7 +175,7 @@ END OF TERMS AND CONDITIONS - Copyright (c) 2014-2021 Megvii Inc. All rights reserved. + Copyright (c) 2014-2023 Megvii Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/dnn/cuda-stub/src/dlopen_helper.h b/dnn/cuda-stub/src/dlopen_helper.h index 842fd525ab18d97ecd851bf7cab00fba0d8507fd..3638489e3b020cdca964f35bb49fe97e641bddd6 100644 --- a/dnn/cuda-stub/src/dlopen_helper.h +++ b/dnn/cuda-stub/src/dlopen_helper.h @@ -115,10 +115,7 @@ static void* resolve_library_func(void* handle, const char* func) { cnt++; //! do not print all annoying msg at broken driver env, for example empty libcuda.so or libcuda.dll if (cnt < 3) { - LOGD("failed to load %s func: %s.(May caused by currently driver is too old, \ - if you find cuda is not available(by import megengine as mge; mge.get_device_count(\"gpu\") \ - or find some inexplicable crash of the program, try upgrade driver from \ - https://developer.nvidia.com/cuda-downloads)", g_default_api_name, func); + LOGD("failed to load %s func: %s.(May caused by currently driver is too old, \nif you find cuda is not available(by import megengine as mge; mge.get_device_count(\"gpu\")\nor find some inexplicable crash of the program,\ntry upgrade driver from https://developer.nvidia.com/cuda-downloads)", g_default_api_name, func); } } return ret; diff --git a/scripts/whl/manylinux2014/Dockerfile_aarch64 b/scripts/whl/manylinux2014/Dockerfile_aarch64 index def83efc1fd07d151985a6849b2120936ce0418a..ffe4320c4741360a8c08d39fea4ae5d664183525 100644 --- a/scripts/whl/manylinux2014/Dockerfile_aarch64 +++ b/scripts/whl/manylinux2014/Dockerfile_aarch64 @@ -8,4 +8,3 @@ RUN ./setup_mirror.sh "$platform" ADD init_image.sh /tmp RUN /tmp/init_image.sh aarch64 && rm -f /tmp/init_image.sh - diff --git a/scripts/whl/manylinux2014/Dockerfile_aarch64_ubuntu2004 b/scripts/whl/manylinux2014/Dockerfile_aarch64_ubuntu2004 new file mode 100644 index 0000000000000000000000000000000000000000..4f2db8beab771bc97f4a2b62d44b668fc4f86945 --- /dev/null +++ b/scripts/whl/manylinux2014/Dockerfile_aarch64_ubuntu2004 @@ -0,0 +1,42 @@ +FROM ubuntu:20.04 + +ENV UID=1024 PATH=${PATH}:/usr/local/cuda/bin + +# update +RUN apt-get update -y + +# build env +RUN DEBIAN_FRONTEND=noninteractive \ +TZ=Etc/UTC apt-get install -y \ +pkg-config build-essential git-lfs \ +autoconf gdb git-core gnupg flex bison gperf \ +zip curl zlib1g-dev gcc \ +g++ rsync wget vim llvm clang + +# install cmake 3.15.2 +RUN cd /tmp ; wget https://cmake.org/files/v3.15/cmake-3.15.2.tar.gz;tar -xzvf cmake-3.15.2.tar.gz;cd cmake-3.15.2;./configure; make -j8; make install + +# install base python3 +RUN apt-get install -y python3 +RUN apt-get install -y python3-dev python3-pip python3-numpy python3-setuptools + +# install pyenv +RUN curl https://pyenv.run | bash + +# install more env for build python3 from src +RUN apt-get install -y checkinstall libncursesw5-dev \ +libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libreadline-dev \ +libffi-dev liblzma-dev libncurses5-dev libdb-dev libexpat-dev libtinfo5 libtinfo-dev + +# install more env for build swig and ninja +RUN apt-get install -y libpcre3 libpcre3-dev re2c + +# call other build env +ADD init_ubuntu2004.sh /tmp +RUN /tmp/init_ubuntu2004.sh && rm -f /tmp/init_ubuntu2004.sh + +# install lsb-release to distinguish is ubuntu or manylinux +RUN apt-get install -y lsb-release + +# install gcc/g++ 8 as gcc/g++ 9 as issue: some code will take about 1h to run as on aarch64-ubuntu +RUN apt-get install -y gcc-8 g++-8 diff --git a/scripts/whl/manylinux2014/build_image.sh b/scripts/whl/manylinux2014/build_image.sh index 9613278711c66ef2f8bee59b07bd955213ba00ed..b06bc58cc09b4d58bdfab3563d269729efcd461c 100755 --- a/scripts/whl/manylinux2014/build_image.sh +++ b/scripts/whl/manylinux2014/build_image.sh @@ -7,3 +7,10 @@ config_docker_file cd $(dirname $0) echo "docker_file is ${docker_file}" docker build -t env_manylinux2014:latest -f ${docker_file} . + +OS=$(uname -m) +if [[ $OS = "aarch64" ]]; then + # as nivida trt-8.5.3.1 build from ubuntu20.04, which depends on new glibc, env_manylinux2014 do not satisfy, so need build aarch64-ubuntu20.04 docker + echo "as in aarch64, need build aarch64-ubuntu20.04 docker" + docker build -t ubuntu2004:latest -f Dockerfile_aarch64_ubuntu2004 . +fi diff --git a/scripts/whl/manylinux2014/build_wheel_common.sh b/scripts/whl/manylinux2014/build_wheel_common.sh index 60f8959e9fe34dc517f3dae269e4e5cc04bc6e24..77da296a1dd0a2f3d2a9bebe488f0b4d0449844e 100755 --- a/scripts/whl/manylinux2014/build_wheel_common.sh +++ b/scripts/whl/manylinux2014/build_wheel_common.sh @@ -13,7 +13,9 @@ TensorRT_LIB_DIR="/opt/tensorrt/lib/" SDK_NAME="unknown" x86_64_support_version="cu101 cu111 cu112 cpu cu111_cudnn821_tensorRT825 cu114 cu118" -aarch64_support_version="cu102_JetsonNano cu111 cpu" +aarch64_support_version="cu102_JetsonNano cu111 cpu cu118" +docker_tag="env_manylinux2014:latest" + if [[ -z ${IN_CI} ]] then IN_CI="false" @@ -236,6 +238,12 @@ elif [ $SDK_NAME == "cu118" ];then CUDNN_LIB_DIR="/opt/cudnn/lib/" CUDA_LIB_DIR="/usr/local/cuda/targets/x86_64-linux/lib/" TensorRT_LIB_DIR="/opt/tensorrt/lib/" + if [ ${machine} == "aarch64" ];then + CUDA_LIB_DIR="/usr/local/cuda/targets/sbsa-linux/lib/" + # aarch64-trt8 libs build from ubuntu2004, which depends on new glibc and libc++ + # manylinux2014 do satify the glibc/libc++ requirement, so we need to use ubuntu2004 as base image + docker_tag="ubuntu2004:latest" + fi CUDA_COPY_LIB_LIST="\ ${CUDA_LIB_DIR}/libnvrtc.so.11.2:\ @@ -262,6 +270,23 @@ elif [ $SDK_NAME == "cu118" ];then -gencode arch=compute_86,code=sm_86 \ -gencode arch=compute_89,code=sm_89 \ -gencode arch=compute_89,code=compute_89\" " + if [ ${machine} == "aarch64" ];then + if [ ${IN_CI} = "true" ]; then + # ci taishan is use SM_75 card + EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON -DMGE_CUDA_GENCODE=\"-gencode arch=compute_75,code=sm_75\" " + else + # support orin and remove 1080Ti + EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \ + -DMGE_CUDA_GENCODE=\" \ + -gencode arch=compute_70,code=sm_70 \ + -gencode arch=compute_75,code=sm_75 \ + -gencode arch=compute_80,code=sm_80 \ + -gencode arch=compute_86,code=sm_86 \ + -gencode arch=compute_87,code=sm_87 \ + -gencode arch=compute_89,code=sm_89 \ + -gencode arch=compute_89,code=compute_89\" " + fi + fi elif [ $SDK_NAME == "cpu" ];then @@ -407,6 +432,10 @@ if [ ${machine} == "aarch64" ];then # infact ubuntu gcc version: gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0 is OK echo "force use gcc8 on aarch64 linux" BUILD_GCC8="ON" + if [ $SDK_NAME == "cu118" ];then + echo "cu118 with aarch64 will build at ubuntu2004 docker env, so do not use gcc8" + BUILD_GCC8="OFF" + fi fi if [ "$BUILD_GCC8" == "ON" ];then @@ -443,4 +472,4 @@ docker run --rm ${docker_args} $TMPFS_ARGS \ -v ${TENSORRT_ROOT_DIR}:/opt/tensorrt \ -v ${BASEDIR}:/home/code \ -v ${OUTPUTDIR}:/home/output:rw \ - env_manylinux2014:latest /bin/bash -c "$run_cmd" + ${docker_tag} /bin/bash -c "$run_cmd" diff --git a/scripts/whl/manylinux2014/do_build_common.sh b/scripts/whl/manylinux2014/do_build_common.sh index 1c862944f557ac95fbd2fc69e8181a3e4f17a5ac..3bf377ea2bc007177e61ffade3a75f9b199ff7b2 100755 --- a/scripts/whl/manylinux2014/do_build_common.sh +++ b/scripts/whl/manylinux2014/do_build_common.sh @@ -1,5 +1,13 @@ #!/bin/bash -e +IN_UBUNTU_DOCKER_ENV="OFF" +if which lsb_release && lsb_release -a | grep "Ubuntu"; then + IN_UBUNTU_DOCKER_ENV="ON" + # some code will take about 1h to run as on aarch64-ubuntu + export CC=gcc-8 + export CXX=g++-8 +fi + function handle_strip() { echo "now handle strip $1" objcopy --only-keep-debug $1 $1.dbg @@ -42,7 +50,9 @@ function patch_elf_depend_lib_mgb_mge() { echo "handle common depend lib for mgb or mge" LIBS_DIR=${BUILD_DIR}/staging/megengine/core/lib mkdir -p ${LIBS_DIR} - cp /usr/lib64/libatomic.so.1 ${LIBS_DIR} + if [ ${IN_UBUNTU_DOCKER_ENV} = "OFF" ]; then + cp /usr/lib64/libatomic.so.1 ${LIBS_DIR} + fi patchelf --remove-rpath ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so patchelf --force-rpath --set-rpath '$ORIGIN/lib' ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so @@ -73,6 +83,10 @@ SRC_DIR=$(readlink -f "`dirname $0`/../../../") source ${SRC_DIR}/scripts/whl/utils/utils.sh SUPPORT_ALL_VERSION="36m 37m 38 39 310" +if [ ${IN_UBUNTU_DOCKER_ENV} = "ON" ]; then + SUPPORT_ALL_VERSION="3.6.10 3.7.7 3.8.3 3.9.4 3.10.1" + echo "in ubuntu docker env, override support all python version: ${SUPPORT_ALL_VERSION}" +fi ALL_PYTHON=${ALL_PYTHON} if [[ -z ${ALL_PYTHON} ]] then @@ -92,6 +106,42 @@ if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/ fi +function config_ubuntu_python_env() { + PYTHON_DIR=~/.pyenv/versions/$1/ + PYTHON_BIN=~/.pyenv/versions/$1/bin + if [ ! -f "$PYTHON_BIN/python3" ]; then + echo "ERR: can not find $PYTHON_BIN , Invalid python package" + echo "now support list: ${FULL_PYTHON_VER}" + err_env + else + echo "put python3 to env..." + export PATH=${PYTHON_BIN}:$PATH + which python3 + fi + echo ${ver} + + if [ "$1" = "3.6.10" ]; then + PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.6m + PYTHON_LIBRARY=${PYTHON_DIR}/lib/libpython3.6m.so + elif [ "$1" = "3.7.7" ]; then + PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.7m + PYTHON_LIBRARY=${PYTHON_DIR}/lib/libpython3.7m.so + elif [[ "$1" = "3.8.3" || "$1" = "3.8.10" ]]; then + PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.8 + PYTHON_LIBRARY=${PYTHON_DIR}/lib/libpython3.8.so + elif [ "$1" = "3.9.4" ]; then + PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.9 + PYTHON_LIBRARY=${PYTHON_DIR}/lib/libpython3.9.so + elif [ "$1" = "3.10.1" ]; then + PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.10 + PYTHON_LIBRARY=${PYTHON_DIR}/lib/libpython3.10.so + else + echo "ERR: DO NOT SUPPORT PYTHON VERSION" + echo "now support list: ${FULL_PYTHON_VER}" + exit -1 + fi +} + # here we just treat cu file should not in the increment build file list INCREMENT_KEY_WORDS=".cu.o is dirty" IS_IN_FIRST_LOOP=TRUE @@ -107,22 +157,42 @@ do rm -rf ${BUILD_DIR} fi - python_ver=`echo $ver | tr -d m` - MAJOR=${python_ver:0:1} - MINOR=${python_ver:1} - PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver} - - SUFFIX= - if [[ $MINOR -lt 8 ]];then - SUFFIX="m" - fi - + # export common args export EXTRA_CMAKE_ARGS="${ORG_EXTRA_CMAKE_FLAG} -DCMAKE_BUILD_TYPE=RelWithDebInfo" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_CUSTOM_OP=ON" + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON" + + if [ ${IN_UBUNTU_DOCKER_ENV} = "ON" ]; then + echo "into Ubuntu env" + # config env + config_ubuntu_python_env ${ver} + #check env + if [ ! -f "$PYTHON_LIBRARY" ]; then + echo "ERR: can not find $PYTHON_LIBRARY , Invalid python package" + err_env + fi + if [ ! -d "$PYTHON_INCLUDE_DIR" ]; then + echo "ERR: can not find $PYTHON_INCLUDE_DIR , Invalid python package" + err_env + fi + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}" + else + echo "into manylinux env" + python_ver=`echo $ver | tr -d m` + MAJOR=${python_ver:0:1} + MINOR=${python_ver:1} + PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver} + + SUFFIX= + if [[ $MINOR -lt 8 ]];then + SUFFIX="m" + fi + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}/include/python${MAJOR}.${MINOR}${SUFFIX}" + fi + #append cmake args for config python export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3" + # please do not config to real python path, it will cause import failed if user python3 do not have libpython3.xx.so[dynamic lib] export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}/lib/" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}/include/python${MAJOR}.${MINOR}${SUFFIX}" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON" if [ -d "${BUILD_DIR}" ]; then # insure rm have args @@ -174,8 +244,15 @@ do cd /home/output mkdir -p ${SRC_DIR}/scripts/whl/manylinux2014/output/wheelhouse/${SDK_NAME} cd ${BUILD_DIR}/staging/dist/ - org_whl_name=`ls Meg*${ver}*.whl` - compat_whl_name=`echo ${org_whl_name} | sed 's/linux/manylinux2014/'` + org_whl_name="null" + if [ ${IN_UBUNTU_DOCKER_ENV} = "ON" ]; then + # ubuntu glibc higher than manylinux2014, so keep org name + org_whl_name=`ls Meg*.whl` + compat_whl_name=${org_whl_name} + else + org_whl_name=`ls Meg*${ver}*.whl` + compat_whl_name=`echo ${org_whl_name} | sed 's/linux/manylinux2014/'` + fi echo "org whl name: ${org_whl_name}" echo "comapt whl name: ${compat_whl_name}" mv ${org_whl_name} ${SRC_DIR}/scripts/whl/manylinux2014/output/wheelhouse/${SDK_NAME}/${compat_whl_name} diff --git a/scripts/whl/manylinux2014/init_ubuntu2004.sh b/scripts/whl/manylinux2014/init_ubuntu2004.sh new file mode 100755 index 0000000000000000000000000000000000000000..25299ac960a3a67639e3ffd9e02573c99504cb82 --- /dev/null +++ b/scripts/whl/manylinux2014/init_ubuntu2004.sh @@ -0,0 +1,54 @@ +#!/bin/bash -e + +ALL_PYTHON="3.6.10 3.7.7 3.8.3 3.9.4 3.10.1" +SWIG_URL='https://codeload.github.com/swig/swig/tar.gz/refs/tags/rel-3.0.12' + +NINJA_URL='https://codeload.github.com/ninja-build/ninja/tar.gz/refs/tags/v1.10.0' +for ver in ${ALL_PYTHON} +do + numpy_version="1.19.5" + if [ ${ver} = "3.10.1" ];then + numpy_version="1.21.6" + fi + echo "Install python ${ver}" + env PYTHON_CONFIGURE_OPTS="--enable-shared" ~/.pyenv/bin/pyenv install ${ver} + ~/.pyenv/versions/${ver}/bin/python3 -m pip install --upgrade pip + ~/.pyenv/versions/${ver}/bin/python3 -m pip install numpy==${numpy_version} setuptools==46.1.3 wheel +done + +pushd /home >/dev/null +echo "Install swig" +curl -sSL ${SWIG_URL} | tar xz +pushd swig-rel-3.0.12 >/dev/null +./autogen.sh +mkdir build +pushd build >/dev/null +../configure +make -j$(nproc) +make install +popd >/dev/null +popd >/dev/null +rm -rf swig-3.0.12 + +echo "Install ninja build" +curl -sSL ${NINJA_URL} | tar xz +pushd ninja-1.10.0 >/dev/null +mkdir build +pushd build >/dev/null +cmake .. -DCMAKE_BUILD_TYPE=Release +make -j$(nproc) +cp ninja /usr/bin/ +popd >/dev/null +popd >/dev/null +rm -rf ninja-1.10.0 +popd >/dev/null + +pushd /tmp >/dev/null +echo "Install patchelf" +curl -sSL https://github.com/NixOS/patchelf/archive/0.12.tar.gz | tar xz +pushd /tmp/patchelf-0.12 >/dev/null +sed -i '331s/32/256/' ./src/patchelf.cc +./bootstrap.sh && ./configure && make install-strip +popd +rm -rf /tmp/patchelf-0.12 +popd diff --git a/scripts/whl/utils/utils.sh b/scripts/whl/utils/utils.sh index 1bf34dac447b27f70b1e7e8b3e7a92f69d56fa59..e8f3ab4d68d4ce6e8f5f4a3df02e262b119a3e54 100755 --- a/scripts/whl/utils/utils.sh +++ b/scripts/whl/utils/utils.sh @@ -78,18 +78,29 @@ function check_build_ninja_python_api() { INCLUDE_KEYWORD="${ver}\\\\include" PYTHON_API_INCLUDES="3.6.8\\\\include 3.7.7\\\\include 3.8.3\\\\include 3.9.4\\\\include 3.10.1\\\\include" elif [[ $OS =~ "Linux" ]]; then - ver=`echo $ver | tr -d m` - INCLUDE_KEYWORD="include/python3.${ver:1}" # like 39/310 - info=`command -v termux-info || true` - if [[ "${info}" =~ "com.termux" ]]; then - echo "find termux-info at: ${info}" + if which lsb_release && lsb_release -a | grep "Ubuntu"; then + echo "into Ubuntu env" is_punctuation=${ver:4:1} INCLUDE_KEYWORD="include/python3.${ver:2:1}" if [ ${is_punctuation} = "." ]; then INCLUDE_KEYWORD="include/python3.${ver:2:2}" fi + PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8 include/python3.9 include/python3.10" + else + echo "into manylinux env" + ver=`echo $ver | tr -d m` + INCLUDE_KEYWORD="include/python3.${ver:1}" # like 39/310 + info=`command -v termux-info || true` + if [[ "${info}" =~ "com.termux" ]]; then + echo "find termux-info at: ${info}" + is_punctuation=${ver:4:1} + INCLUDE_KEYWORD="include/python3.${ver:2:1}" + if [ ${is_punctuation} = "." ]; then + INCLUDE_KEYWORD="include/python3.${ver:2:2}" + fi + fi + PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8 include/python3.9 include/python3.10" fi - PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8 include/python3.9 include/python3.10" elif [[ $OS =~ "Darwin" ]]; then is_punctuation=${ver:4:1} INCLUDE_KEYWORD="include/python3.${ver:2:1}"