From 48f0c7070a59a0a0716ff4e006f407cac612fa9c Mon Sep 17 00:00:00 2001 From: "C.J. Jameson" Date: Tue, 19 Sep 2017 16:36:42 -0700 Subject: [PATCH] Enable concourse to build GPDB on Ubuntu - Open Source GPDB compile - No ORCA yet -- getting there soon [#151180202] Signed-off-by: Goutam Tadi --- README.linux.md | 30 ++++++++++------- README.md | 2 +- concourse/pipelines/pipeline.yml | 24 +++++++++++++- concourse/pipelines/pr_pipeline.yml | 2 +- ...h => compile_gpdb_open_source_centos.bash} | 0 .../compile_gpdb_open_source_ubuntu.bash | 32 +++++++++++++++++++ concourse/scripts/setup_gpadmin_user.bash | 19 +++++++++-- concourse/tasks/compile_gpdb_open_source.yml | 10 +----- .../tasks/compile_gpdb_open_source_centos.yml | 9 ++++++ .../tasks/compile_gpdb_open_source_ubuntu.yml | 9 ++++++ 10 files changed, 111 insertions(+), 26 deletions(-) rename concourse/scripts/{compile_gpdb_open_source.bash => compile_gpdb_open_source_centos.bash} (100%) create mode 100755 concourse/scripts/compile_gpdb_open_source_ubuntu.bash mode change 100644 => 120000 concourse/tasks/compile_gpdb_open_source.yml create mode 100644 concourse/tasks/compile_gpdb_open_source_centos.yml create mode 100644 concourse/tasks/compile_gpdb_open_source_ubuntu.yml diff --git a/README.linux.md b/README.linux.md index eaf7219a71..7ccfd6ffc6 100644 --- a/README.linux.md +++ b/README.linux.md @@ -1,20 +1,26 @@ -1. ORCA require [CMake](https://cmake.org), make sure you have it installed. +1. ORCA requires [CMake](https://cmake.org); make sure you have it installed. Installation instructions vary, please check the CMake website. 1. Install needed python modules - Add the following Python modules (2.7 & 2.6 are supported) + Add the following Python modules (2.7 & 2.6 are supported) - * psutil - * lockfile (>= 0.9.1) - * paramiko - * setuptools + * psutil + * lockfile (>= 0.9.1) + * paramiko + * setuptools - If necessary, upgrade modules using "pip install --upgrade". - pip should be at least version 7.x.x. + If necessary, upgrade modules using "pip install --upgrade". + pip should be at least version 7.x.x. -2. Verify that you can ssh to your machine name without a password -``` -ssh # e.g., ssh briarwood -``` +1. Verify that you can ssh to your machine name without a password + ``` + ssh # e.g., ssh briarwood + ``` + +1. If you are using CentOS, first make sure that you add `/usr/local/lib` and + `/usr/local/lib64` to `/etc/ld.so.conf`, then run command `ldconfig`. + + If you are using Ubuntu, confirm that `/usr/local/lib` is included via + `/etc/ld.so.conf`, then run command `ldconfig`. diff --git a/README.md b/README.md index af86fd730e..751339ae2c 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Follow the directions in the [ORCA README](https://github.com/greenplum-db/gporc checking Checking ORCA version... configure: error: Your ORCA version is expected to be 2.33.XXX ### Build the database -Note: If you are using CentOS, first make sure that you add `/usr/local/lib` and `/usr/local/lib64` to `/etc/ld.so.conf`, run command `ldconfig`. + ``` # Configure build environment to install at /usr/local/gpdb ./configure --with-perl --with-python --with-libxml --prefix=/usr/local/gpdb diff --git a/concourse/pipelines/pipeline.yml b/concourse/pipelines/pipeline.yml index 1c66c27987..d23f494161 100644 --- a/concourse/pipelines/pipeline.yml +++ b/concourse/pipelines/pipeline.yml @@ -6,6 +6,7 @@ groups: - compile_gpdb_binary_swap_centos6 - compile_gpdb_centos7 - compile_gpdb_sles11 + - compile_gpdb_ubuntu16 - compile_gpdb_windows_cl - compile_gpdb_aix7_remote - icw_planner_centos6 @@ -188,6 +189,12 @@ resources: repository: pivotaldata/sles-gpdb-dev tag: 11-beta +- name: ubuntu-gpdb-dev-16 + type: docker-image + source: + repository: pivotaldata/ubuntu-gpdb-dev + tag: '16.04' + - name: centos-mingw type: docker-image source: @@ -582,7 +589,7 @@ jobs: - get: centos-gpdb-dev-6 - task: compile_gpdb image: centos-gpdb-dev-6 - file: gpdb_src/concourse/tasks/compile_gpdb_open_source.yml + file: gpdb_src/concourse/tasks/compile_gpdb_open_source_centos.yml params: CONFIGURE_FLAGS: {{configure_flags}} @@ -620,6 +627,20 @@ jobs: params: file: gpdb_artifacts/greenplum-loaders-*-rhel7-x86_64.zip +- name: compile_gpdb_ubuntu16 + plan: + - aggregate: + - get: reduced-frequency-trigger + trigger: {{reduced-frequency-trigger-flag}} + - get: gpdb_src + trigger: {{gpdb_src-trigger-flag}} + - get: ubuntu-gpdb-dev-16 + - task: compile_gpdb + image: ubuntu-gpdb-dev-16 + file: gpdb_src/concourse/tasks/compile_gpdb_open_source_ubuntu.yml + params: + CONFIGURE_FLAGS: {{configure_flags}} + - name: compile_gpdb_sles11 plan: - aggregate: @@ -1907,6 +1928,7 @@ jobs: - compile_gpdb_open_source_centos6 - compile_gpdb_centos7 - compile_gpdb_sles11 + - compile_gpdb_ubuntu16 - compile_gpdb_windows_cl - compile_gpdb_aix7_remote - icw_planner_centos6 diff --git a/concourse/pipelines/pr_pipeline.yml b/concourse/pipelines/pr_pipeline.yml index be56293590..d0436458d5 100644 --- a/concourse/pipelines/pr_pipeline.yml +++ b/concourse/pipelines/pr_pipeline.yml @@ -55,7 +55,7 @@ jobs: - aggregate: - task: compile_gpdb_open_source_centos6 - file: gpdb_pr/concourse/tasks/compile_gpdb_open_source.yml + file: gpdb_pr/concourse/tasks/compile_gpdb_open_source_centos.yml image: centos-gpdb-dev-6 input_mapping: gpdb_src: gpdb_pr diff --git a/concourse/scripts/compile_gpdb_open_source.bash b/concourse/scripts/compile_gpdb_open_source_centos.bash similarity index 100% rename from concourse/scripts/compile_gpdb_open_source.bash rename to concourse/scripts/compile_gpdb_open_source_centos.bash diff --git a/concourse/scripts/compile_gpdb_open_source_ubuntu.bash b/concourse/scripts/compile_gpdb_open_source_ubuntu.bash new file mode 100755 index 0000000000..184a647927 --- /dev/null +++ b/concourse/scripts/compile_gpdb_open_source_ubuntu.bash @@ -0,0 +1,32 @@ +#!/bin/bash -l +set -exo pipefail + +GREENPLUM_INSTALL_DIR=/usr/local/gpdb + +# This is a canonical way to build GPDB. The intent is to validate that GPDB compiles +# with a fairly basic build. It is not meant to be exhaustive or include all features +# and components available in GPDB. + +function build_gpdb() { + pushd gpdb_src + CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --with-perl --with-libxml \ + --disable-orca --with-python --disable-gpfdist --prefix=${GREENPLUM_INSTALL_DIR} + # Use -j4 to speed up the build. (Doesn't seem worth trying to guess a better + # value based on number of CPUs or anything like that. Going above -j4 wouldn't + # make it much faster, and -j4 is small enough to not hurt too badly even on + # a single-CPU system + make -j4 + make install + popd +} + +function unittest_check_gpdb() { + make -C gpdb_src/src/backend -s unittest-check +} + +function _main() { + build_gpdb + unittest_check_gpdb +} + +_main "$@" diff --git a/concourse/scripts/setup_gpadmin_user.bash b/concourse/scripts/setup_gpadmin_user.bash index 560b8f696d..0d4c00326d 100755 --- a/concourse/scripts/setup_gpadmin_user.bash +++ b/concourse/scripts/setup_gpadmin_user.bash @@ -62,6 +62,9 @@ setup_gpadmin_user() { centos) /usr/sbin/useradd -G supergroup,tty gpadmin ;; + ubuntu) + /usr/sbin/useradd -G supergroup,tty gpadmin + ;; *) echo "Unknown OS: $TEST_OS"; exit 1 ;; esac echo -e "password\npassword" | passwd gpadmin @@ -71,7 +74,9 @@ setup_gpadmin_user() { } setup_sshd() { - test -e /etc/ssh/ssh_host_key || ssh-keygen -f /etc/ssh/ssh_host_key -N '' -t rsa1 + if [ ! "$TEST_OS" = 'ubuntu' ]; then + test -e /etc/ssh/ssh_host_key || ssh-keygen -f /etc/ssh/ssh_host_key -N '' -t rsa1 + fi test -e /etc/ssh/ssh_host_rsa_key || ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa test -e /etc/ssh/ssh_host_dsa_key || ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa @@ -86,7 +91,13 @@ setup_sshd() { setup_ssh_for_user root - /usr/sbin/sshd + # Test that sshd can start + if [ -x /etc/init.d/sshd ]; then + /etc/init.d/sshd start + elif [ -x /etc/init.d/ssh ]; then + # Ubuntu uses ssh instead of sshd + /etc/init.d/ssh start + fi ssh_keyscan_for_user root ssh_keyscan_for_user gpadmin @@ -101,6 +112,10 @@ determine_os() { echo "sles" return fi + if lsb_release -a | grep -q 'Ubuntu' ; then + echo "ubuntu" + return + fi echo "Could not determine operating system type" >/dev/stderr exit 1 } diff --git a/concourse/tasks/compile_gpdb_open_source.yml b/concourse/tasks/compile_gpdb_open_source.yml deleted file mode 100644 index b23759a180..0000000000 --- a/concourse/tasks/compile_gpdb_open_source.yml +++ /dev/null @@ -1,9 +0,0 @@ -platform: linux -image_resource: - type: docker-image -inputs: - - name: gpdb_src -run: - path: gpdb_src/concourse/scripts/compile_gpdb_open_source.bash -params: - CONFIGURE_FLAGS: "" diff --git a/concourse/tasks/compile_gpdb_open_source.yml b/concourse/tasks/compile_gpdb_open_source.yml new file mode 120000 index 0000000000..0a0b03a186 --- /dev/null +++ b/concourse/tasks/compile_gpdb_open_source.yml @@ -0,0 +1 @@ +concourse/tasks/compile_gpdb_open_source_centos.yml \ No newline at end of file diff --git a/concourse/tasks/compile_gpdb_open_source_centos.yml b/concourse/tasks/compile_gpdb_open_source_centos.yml new file mode 100644 index 0000000000..e77cc04620 --- /dev/null +++ b/concourse/tasks/compile_gpdb_open_source_centos.yml @@ -0,0 +1,9 @@ +platform: linux +image_resource: + type: docker-image +inputs: + - name: gpdb_src +run: + path: gpdb_src/concourse/scripts/compile_gpdb_open_source_centos.bash +params: + CONFIGURE_FLAGS: "" diff --git a/concourse/tasks/compile_gpdb_open_source_ubuntu.yml b/concourse/tasks/compile_gpdb_open_source_ubuntu.yml new file mode 100644 index 0000000000..11bba3e3c3 --- /dev/null +++ b/concourse/tasks/compile_gpdb_open_source_ubuntu.yml @@ -0,0 +1,9 @@ +platform: linux +image_resource: + type: docker-image +inputs: + - name: gpdb_src +run: + path: gpdb_src/concourse/scripts/compile_gpdb_open_source_ubuntu.bash +params: + CONFIGURE_FLAGS: "" -- GitLab