Enable concourse to build GPDB on Ubuntu

- Open Source GPDB compile
- No ORCA yet -- getting there soon

[#151180202]
Signed-off-by: NGoutam Tadi <gtadi@pivotal.io>

README.linux.md

-1. ORCA require [CMake](https://cmake.org), make sure you have it installed.
+1. ORCA requires [CMake](https://cmake.org); make sure you have it installed.
    Installation instructions vary, please check the CMake website.
    
 1. Install needed python modules
 
-  Add the following Python modules (2.7 & 2.6 are supported)
+    Add the following Python modules (2.7 & 2.6 are supported)
 
-  * psutil
-  * lockfile (>= 0.9.1)
-  * paramiko
-  * setuptools
+    * psutil
+    * lockfile (>= 0.9.1)
+    * paramiko
+    * setuptools
 
-  If necessary, upgrade modules using "pip install --upgrade".
-  pip should be at least version 7.x.x.
+    If necessary, upgrade modules using "pip install --upgrade".
+    pip should be at least version 7.x.x.
 
-2. Verify that you can ssh to your machine name without a password
-```
-ssh <hostname of your machine>  # e.g., ssh briarwood
-```
+1. Verify that you can ssh to your machine name without a password
 
+    ```
+    ssh <hostname of your machine>  # e.g., ssh briarwood
+    ```
+
+1. If you are using CentOS, first make sure that you add `/usr/local/lib` and
+   `/usr/local/lib64` to `/etc/ld.so.conf`, then run command `ldconfig`.
+
+    If you are using Ubuntu, confirm that `/usr/local/lib` is included via
+	`/etc/ld.so.conf`, then run command `ldconfig`.

README.md

@@ -54,7 +54,7 @@ Follow the directions in the [ORCA README](https://github.com/greenplum-db/gporc
     checking Checking ORCA version... configure: error: Your ORCA version is expected to be 2.33.XXX
 
 ### Build the database
-Note: If you are using CentOS, first make sure that you add `/usr/local/lib` and `/usr/local/lib64` to `/etc/ld.so.conf`, run command `ldconfig`.
+
 ```
 # Configure build environment to install at /usr/local/gpdb
 ./configure --with-perl --with-python --with-libxml --prefix=/usr/local/gpdb

concourse/pipelines/pipeline.yml

@@ -6,6 +6,7 @@ groups:
   - compile_gpdb_binary_swap_centos6
   - compile_gpdb_centos7
   - compile_gpdb_sles11
+  - compile_gpdb_ubuntu16
   - compile_gpdb_windows_cl
   - compile_gpdb_aix7_remote
   - icw_planner_centos6
@@ -188,6 +189,12 @@ resources:
     repository: pivotaldata/sles-gpdb-dev
     tag: 11-beta
 
+- name: ubuntu-gpdb-dev-16
+  type: docker-image
+  source:
+    repository: pivotaldata/ubuntu-gpdb-dev
+    tag: '16.04'
+
 - name: centos-mingw
   type: docker-image
   source:
@@ -582,7 +589,7 @@ jobs:
     - get: centos-gpdb-dev-6
   - task: compile_gpdb
     image: centos-gpdb-dev-6
-    file: gpdb_src/concourse/tasks/compile_gpdb_open_source.yml
+    file: gpdb_src/concourse/tasks/compile_gpdb_open_source_centos.yml
     params:
         CONFIGURE_FLAGS: {{configure_flags}}
 
@@ -620,6 +627,20 @@ jobs:
       params:
         file: gpdb_artifacts/greenplum-loaders-*-rhel7-x86_64.zip
 
+- name: compile_gpdb_ubuntu16
+  plan:
+  - aggregate:
+    - get: reduced-frequency-trigger
+      trigger: {{reduced-frequency-trigger-flag}}
+    - get: gpdb_src
+      trigger: {{gpdb_src-trigger-flag}}
+    - get: ubuntu-gpdb-dev-16
+  - task: compile_gpdb
+    image: ubuntu-gpdb-dev-16
+    file: gpdb_src/concourse/tasks/compile_gpdb_open_source_ubuntu.yml
+    params:
+      CONFIGURE_FLAGS: {{configure_flags}}
+
 - name: compile_gpdb_sles11
   plan:
   - aggregate:
@@ -1907,6 +1928,7 @@ jobs:
     - compile_gpdb_open_source_centos6
     - compile_gpdb_centos7
     - compile_gpdb_sles11
+    - compile_gpdb_ubuntu16
     - compile_gpdb_windows_cl
     - compile_gpdb_aix7_remote
     - icw_planner_centos6

concourse/pipelines/pr_pipeline.yml

@@ -55,7 +55,7 @@ jobs:
 
     - aggregate:
       - task: compile_gpdb_open_source_centos6
-        file: gpdb_pr/concourse/tasks/compile_gpdb_open_source.yml
+        file: gpdb_pr/concourse/tasks/compile_gpdb_open_source_centos.yml
         image: centos-gpdb-dev-6
         input_mapping:
           gpdb_src: gpdb_pr

concourse/scripts/compile_gpdb_open_source_ubuntu.bash

+#!/bin/bash -l
+set -exo pipefail
+
+GREENPLUM_INSTALL_DIR=/usr/local/gpdb
+
+# This is a canonical way to build GPDB. The intent is to validate that GPDB compiles
+# with a fairly basic build. It is not meant to be exhaustive or include all features
+# and components available in GPDB.
+
+function build_gpdb() {
+  pushd gpdb_src
+    CC=$(which gcc) CXX=$(which g++) ./configure --enable-mapreduce --with-perl --with-libxml \
+	--disable-orca --with-python --disable-gpfdist --prefix=${GREENPLUM_INSTALL_DIR}
+    # Use -j4 to speed up the build. (Doesn't seem worth trying to guess a better
+    # value based on number of CPUs or anything like that. Going above -j4 wouldn't
+    # make it much faster, and -j4 is small enough to not hurt too badly even on
+    # a single-CPU system
+    make -j4
+    make install
+  popd
+}
+
+function unittest_check_gpdb() {
+  make -C gpdb_src/src/backend -s unittest-check
+}
+
+function _main() {
+  build_gpdb
+  unittest_check_gpdb
+}
+
+_main "$@"

concourse/scripts/setup_gpadmin_user.bash

@@ -62,6 +62,9 @@ setup_gpadmin_user() {
     centos)
       /usr/sbin/useradd -G supergroup,tty gpadmin
       ;;
+    ubuntu)
+      /usr/sbin/useradd -G supergroup,tty gpadmin
+      ;;
     *) echo "Unknown OS: $TEST_OS"; exit 1 ;;
   esac
   echo -e "password\npassword" | passwd gpadmin
@@ -71,7 +74,9 @@ setup_gpadmin_user() {
 }
 
 setup_sshd() {
-  test -e /etc/ssh/ssh_host_key || ssh-keygen -f /etc/ssh/ssh_host_key -N '' -t rsa1
+  if [ ! "$TEST_OS" = 'ubuntu' ]; then
+    test -e /etc/ssh/ssh_host_key || ssh-keygen -f /etc/ssh/ssh_host_key -N '' -t rsa1
+  fi
   test -e /etc/ssh/ssh_host_rsa_key || ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa
   test -e /etc/ssh/ssh_host_dsa_key || ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa
 
@@ -86,7 +91,13 @@ setup_sshd() {
 
   setup_ssh_for_user root
 
-  /usr/sbin/sshd
+  # Test that sshd can start
+  if [ -x /etc/init.d/sshd ]; then
+    /etc/init.d/sshd start
+  elif [ -x /etc/init.d/ssh ]; then
+    # Ubuntu uses ssh instead of sshd
+    /etc/init.d/ssh start
+  fi
 
   ssh_keyscan_for_user root
   ssh_keyscan_for_user gpadmin
@@ -101,6 +112,10 @@ determine_os() {
     echo "sles"
     return
   fi
+  if lsb_release -a | grep -q 'Ubuntu' ; then
+    echo "ubuntu"
+    return
+  fi
   echo "Could not determine operating system type" >/dev/stderr
   exit 1
 }

concourse/tasks/compile_gpdb_open_source.yml

-platform: linux
-image_resource:
-  type: docker-image
-inputs:
-  - name: gpdb_src
-run:
-  path: gpdb_src/concourse/scripts/compile_gpdb_open_source.bash
-params:
-  CONFIGURE_FLAGS: ""

concourse/tasks/compile_gpdb_open_source.yml

+concourse/tasks/compile_gpdb_open_source_centos.yml
\ No newline at end of file

concourse/tasks/compile_gpdb_open_source_centos.yml

+platform: linux
+image_resource:
+  type: docker-image
+inputs:
+  - name: gpdb_src
+run:
+  path: gpdb_src/concourse/scripts/compile_gpdb_open_source_centos.bash
+params:
+  CONFIGURE_FLAGS: ""

concourse/tasks/compile_gpdb_open_source_ubuntu.yml

+platform: linux
+image_resource:
+  type: docker-image
+inputs:
+  - name: gpdb_src
+run:
+  path: gpdb_src/concourse/scripts/compile_gpdb_open_source_ubuntu.bash
+params:
+  CONFIGURE_FLAGS: ""