diff --git a/README.CentOS.bash b/README.CentOS.bash index c66d5de8ed83a780c4f06333226e66fbafe5ed5e..460e8e8fa2fb36b2b7388aed60f37506d8fb053e 100755 --- a/README.CentOS.bash +++ b/README.CentOS.bash @@ -1,28 +1,31 @@ #!/bin/bash -# TODO: This currently has non-Orca dependencies only. Please expand if you need to build Orca. - # Install needed packages. Please add to this list if you discover additional prerequisites. -sudo yum install -y apr-devel -sudo yum install -y bison -sudo yum install -y bzip2-devel -sudo yum install -y flex -sudo yum install -y gcc -sudo yum install -y libcurl-devel -sudo yum install -y libevent-devel -sudo yum install -y libkadm5 -sudo yum install -y libxml2-devel -sudo yum install -y perl-ExtUtils-Embed -sudo yum install -y python-devel -sudo yum install -y python-paramiko -sudo yum install -y python-psutil -sudo yum install -y python-setuptools -sudo yum install -y readline-devel -sudo yum install -y zlib-devel - -# Install pip sudo yum install -y epel-release -sudo yum install -y python-pip +sudo yum install -y \ + apr-devel \ + bison \ + bzip2-devel \ + cmake3 \ + flex \ + gcc \ + gcc-c++ \ + krb5-devel \ + libcurl-devel \ + libevent-devel \ + libkadm5 \ + libyaml-devel \ + libxml2-devel \ + perl-ExtUtils-Embed \ + python-devel \ + python-paramiko \ + python-pip \ + python-psutil \ + python-setuptools \ + readline-devel \ + xerces-c-devel \ + zlib-devel + # Install lockfile with pip because the yum package `python-pip` is too old (0.8). -sudo pip install lockfile +sudo pip install lockfile conan diff --git a/README.CentOS.md b/README.CentOS.md deleted file mode 100644 index 806c3683cd18d46ce92b331effacfa9ecb428ead..0000000000000000000000000000000000000000 --- a/README.CentOS.md +++ /dev/null @@ -1,37 +0,0 @@ -### CentOS Build Setup Instructions - -These setup instructions only apply to builds without Orca. - -### Install Dependencies - -```bash -./README.CentOS.bash -``` - -### Get an Updated GCC on CentOS 6 - -Install [devtoolset-6][devtoolset-6], which contains gcc 6.3. - -[devtoolset-6]: https://www.softwarecollections.org/en/scls/rhscl/devtoolset-6/ - -First, enable the repository: - -```bash -#CentOS 6: -sudo yum install centos-release-scl -#RHEL on AWS: -sudo yum-config-manager --enable rhui-REGION-rhel-server-rhscl -``` - -Then install devtoolset-6: - -```bash -sudo yum install devtoolset-6-toolchain -``` - -Now running `scl enable devtoolset-6 bash` starts a bash session that puts gcc -6.3 in `$PATH`. To enable devtoolset-6 in all your login shells: - -```bash -echo 'source scl_source enable devtoolset-6' >> ~/.bashrc -``` diff --git a/README.debian b/README.debian deleted file mode 100644 index ff9b626da4d108b15b5fc0a426b42b531e437fba..0000000000000000000000000000000000000000 --- a/README.debian +++ /dev/null @@ -1,62 +0,0 @@ -create ssh keys so you can do ssh localhost without a password -ie ssh-keygen -then cp ~/.ssh/id_rsa.pub authorized_keys - -apt-get install -y git-core -apt-get install -y gcc g++ -apt-get install -y ccache -apt-get install -y libreadline-dev -apt-get install -y bison flex -apt-get install -y zlib1g-dev -apt-get install -y openssl libssl-dev -apt-get install -y libpam-dev -apt-get install -y libcurl4-dev -apt-get install -y libbz2-dev -apt-get install -y python-dev -apt-get install -y ssh -apt-get install -y libapr1-dev -apt-get install -y libevent-dev -apt-get install -y libxml2-dev -apt-get install -y libperl-dev - -apt-get install -y libcurl4-dev -Package libcurl4-dev is a virtual package provided by: -libcurl4-openssl-dev 7.38.0-4+deb8u2 -libcurl4-nss-dev 7.38.0-4+deb8u2 -libcurl4-gnutls-dev 7.38.0-4+deb8u2 - -apt-get install -y python-pip - -pip install lockfile -pip install paramiko -pip install setuptools -pip install psutil -Note: debian8 required pip install --pre psutil - -sysctl.conf -kernel.shmmax = 500000000 -kernel.shmmni = 4096 -kernel.shmall = 4000000000 -kernel.sem = 250 512000 100 2048 -kernel.sysrq = 1 -kernel.core_uses_pid = 1 -kernel.msgmnb = 65536 -kernel.msgmax = 65536 -kernel.msgmni = 2048 -net.ipv4.tcp_syncookies = 1 -net.ipv4.ip_forward = 0 -net.ipv4.conf.default.accept_source_route = 0 -net.ipv4.tcp_tw_recycle = 1 -net.ipv4.tcp_max_syn_backlog = 4096 -net.ipv4.conf.all.arp_filter = 1 -net.ipv4.ip_local_port_range = 1025 65535 -net.core.netdev_max_backlog = 10000 -net.core.rmem_max = 2097152 -net.core.wmem_max = 2097152 -vm.overcommit_memory = 2 - -limits.conf -* soft nofile 65536 -* hard nofile 65536 -* soft nproc 131072 -* hard nproc 131072 diff --git a/README.linux.md b/README.linux.md index 7ccfd6ffc697a23a4ba230754a4613fcba707d03..346899aa70ce9cd038403598f783164c863705e2 100644 --- a/README.linux.md +++ b/README.linux.md @@ -1,26 +1,125 @@ -1. ORCA requires [CMake](https://cmake.org); make sure you have it installed. +## For CentOS: + +- Install Dependencies + + ```bash + ./README.CentOS.bash + ``` + +- If you want to link cmake3 to cmake, run: + + ```bash + ln -sf ../../bin/cmake3 /usr/local/bin/cmake + ``` + +- Make sure that you add `/usr/local/lib` and `/usr/local/lib64` to +`/etc/ld.so.conf`, then run command `ldconfig`. + +- If you want to install and use gcc-6 by default, run: + + ```bash + sudo yum install -y centos-release-scl devtoolset-6-toolchain + echo 'source scl_source enable devtoolset-6' >> ~/.bashrc + ``` + +## For RHEL + +Use dependency script for CentOS. + +- If you want to install `devtoolset-6`: + + ```bash + sudo yum-config-manager --enable rhui-REGION-rhel-server-rhscl + sudo yum install -y devtoolset-6-toolchain + ``` + +## For Ubuntu: + +- Install Dependencies + When you run the README.ubuntu.bash script for dependencies, you will be asked to configure realm for kerberos. + You can enter any realm, since this is just for testing, and during testing, it will reconfigure a local server/client. + If you want to skip this manual configuration, use: + `export DEBIAN_FRONTEND=noninteractive` + + ```bash + ./README.ubuntu.bash + ``` + +- If you want to use gcc-6 and g++-6: + + ```bash + add-apt-repository ppa:ubuntu-toolchain-r/test -y + apt-get update + apt-get install -y gcc-6 g++-6 + ``` + +## Common Platform Tasks: + +Make sure that you add `/usr/local/lib` to `/etc/ld.so.conf`, +then run command `ldconfig`. + +1. ORCA requires [CMake](https://cmake.org) 3.x; make sure you have it installed. Installation instructions vary, please check the CMake website. - -1. Install needed python modules - Add the following Python modules (2.7 & 2.6 are supported) +1. Create gpadmin and setup ssh keys + Either use: + + ```bash + # Requires gpdb clone to be named gpdb_src + gpdb_src/concourse/scripts/setup_gpadmin_user.bash + ``` + to create the gpadmin user and set up keys, - * psutil - * lockfile (>= 0.9.1) - * paramiko - * setuptools + OR - If necessary, upgrade modules using "pip install --upgrade". - pip should be at least version 7.x.x. + manually create ssh keys so you can do ssh localhost without a password, e.g., using ssh-keygen + then cp ~/.ssh/id_rsa.pub authorized_keys 1. Verify that you can ssh to your machine name without a password - ``` + ```bash ssh # e.g., ssh briarwood - ``` + ``` + +1. Set up your system configuration: + + ```bash + + cat >> /etc/sysctl.conf <<-EOF + kernel.shmmax = 500000000 + kernel.shmmni = 4096 + kernel.shmall = 4000000000 + kernel.sem = 250 512000 100 2048 + kernel.sysrq = 1 + kernel.core_uses_pid = 1 + kernel.msgmnb = 65536 + kernel.msgmax = 65536 + kernel.msgmni = 2048 + net.ipv4.tcp_syncookies = 1 + net.ipv4.ip_forward = 0 + net.ipv4.conf.default.accept_source_route = 0 + net.ipv4.tcp_tw_recycle = 1 + net.ipv4.tcp_max_syn_backlog = 4096 + net.ipv4.conf.all.arp_filter = 1 + net.ipv4.ip_local_port_range = 1025 65535 + net.core.netdev_max_backlog = 10000 + net.core.rmem_max = 2097152 + net.core.wmem_max = 2097152 + vm.overcommit_memory = 2 + + EOF + + cat >> /etc/security/limits.conf <<-EOF + * soft nofile 65536 + * hard nofile 65536 + * soft nproc 131072 + * hard nproc 131072 + + EOF + + cat >> /etc/ld.so.conf <<-EOF + /usr/local/lib -1. If you are using CentOS, first make sure that you add `/usr/local/lib` and - `/usr/local/lib64` to `/etc/ld.so.conf`, then run command `ldconfig`. + EOF - If you are using Ubuntu, confirm that `/usr/local/lib` is included via - `/etc/ld.so.conf`, then run command `ldconfig`. + ``` diff --git a/README.md b/README.md index 20d1c023fd71bdb90471f8a85ed8ed883c6b3815..8e2d2885011ef4aaa25b59e8fa3f8f81a47567f0 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ to the segments, and collects the results. Follow [these macOS steps](README.macOS.md) for getting your system ready for GPDB ### Installing dependencies (for Linux developers) -Follow [these linux steps](README.linux.md) for getting your system ready for GPDB +Follow [appropriate linux steps](README.linux.md) for getting your system ready for GPDB ### Build the optimizer diff --git a/README.ubuntu.bash b/README.ubuntu.bash new file mode 100644 index 0000000000000000000000000000000000000000..8e5d6ce1eacc43bd5eca76e08699e0932df2d5a6 --- /dev/null +++ b/README.ubuntu.bash @@ -0,0 +1,44 @@ +#!/bin/bash + +apt-get update +apt-get install -y \ + bison \ + ccache \ + cmake \ + curl \ + flex \ + git-core \ + gcc \ + g++ \ + inetutils-ping \ + krb5-kdc \ + krb5-admin-server \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libkrb5-dev \ + libpam-dev \ + libperl-dev \ + libreadline-dev \ + libssl-dev \ + libxerces-c-dev \ + libxml2-dev \ + libyaml-dev \ + locales \ + net-tools \ + ninja-build \ + openssh-client \ + openssh-server \ + openssl \ + python-dev \ + python-lockfile \ + python-paramiko \ + python-pip \ + python-psutil \ + python-yaml \ + zlib1g-dev + + +pip install conan +