Trigger gpexpand after ICW.

The pipeline job icw_planner_centos7_online_expand tests gpexpand in
such a way, it creates a cluster with only 2 primary+mirror segment
pairs, then expand it to 3 and run ICW to verify the expanded cluster
works correctly.

Now we further expand the cluster size from 3 to 4 after ICW, to verify
that tables can be successfully redistributed.

Also renamed the job to icw_planner_centos7_expand.

concourse/pipelines/gpdb_master-generated.yml

@@ -45,6 +45,7 @@ groups:
   - pg_upgrade
   - icw_gporca_centos7
   - icw_planner_centos7
+  - icw_planner_centos7_expand
   - icw_gporca_sles11
   - icw_gporca_sles12
   - icw_planner_sles12
@@ -129,6 +130,7 @@ groups:
   - pg_upgrade
   - icw_gporca_centos7
   - icw_planner_centos7
+  - icw_planner_centos7_expand
   - compile_gpdb_centos7
   - icw_gporca_sles11
   - icw_gporca_sles12
@@ -1108,6 +1110,27 @@ jobs:
       TEST_BINARY_SWAP: false
       CONFIGURE_FLAGS: {{configure_flags}}
 
+- name: icw_planner_centos7_expand
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed: [compile_gpdb_centos7]
+    - get: bin_gpdb
+      passed: [compile_gpdb_centos7]
+      resource: bin_gpdb_centos7
+      trigger: true
+    - get: gpdb6-centos7-test
+  - task: ic_gpdb
+    file: gpdb_src/concourse/tasks/ic_gpdb.yml
+    image: gpdb6-centos7-test
+    params:
+      MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world
+      TEST_OS: centos
+      TEST_BINARY_SWAP: false
+      CONFIGURE_FLAGS: {{configure_flags}}
+      EXCLUDE_TESTS: instr_in_shmem_setup,instr_in_shmem_terminate,segspace_setup,segspace,segspace_cleanup,cursor,instr_in_shmem,instr_in_shmem_verify,gdd/prepare,gdd/dist-deadlock-01,gdd/dist-deadlock-04,gdd/dist-deadlock-05,gdd/dist-deadlock-06,gdd/dist-deadlock-07,gdd/dist-deadlock-102,gdd/dist-deadlock-103,gdd/dist-deadlock-104,gdd/dist-deadlock-106,gdd/non-lock-105,gdd/non-lock-107,gdd/avoid-qd-deadlock,gdd/prepare-for-local,gdd/local-deadlock-03,gdd/end,concurrent_update_distkeys,concurrent_update_epq
+      TEST_EXPAND: true
+
 - name: icw_gporca_sles11
   plan:
   - aggregate:
@@ -2123,6 +2146,7 @@ jobs:
       - icw_gporca_centos6
       - icw_gporca_centos7
       - icw_planner_centos7
+      - icw_planner_centos7_expand
  ##     - icw_gporca_sles11
  ##     - icw_gporca_sles12
  ##     - icw_planner_sles12
@@ -2189,6 +2213,7 @@ jobs:
       - compile_gpdb_centos7
       - icw_gporca_centos7
       - icw_planner_centos7
+      - icw_planner_centos7_expand
       - resource_group_centos7
  ##     - MADlib_Test_orca_centos7
  ##     - MADlib_Test_planner_centos7

concourse/pipelines/templates/gpdb-tpl.yml

@@ -52,6 +52,7 @@ groups:
 {% if "centos7" in os_types %}
   - icw_gporca_centos7
   - icw_planner_centos7
+  - icw_planner_centos7_expand
 {% endif %}
 {% if "sles" in os_types %}
   - icw_gporca_sles11
@@ -189,6 +190,7 @@ groups:
 {% if "centos7" in os_types %}
   - icw_gporca_centos7
   - icw_planner_centos7
+  - icw_planner_centos7_expand
   - compile_gpdb_centos7
 {% endif %}
 {% if "sles" in os_types %}
@@ -1271,6 +1273,27 @@ jobs:
       TEST_BINARY_SWAP: false
       CONFIGURE_FLAGS: {{configure_flags}}
 
+- name: icw_planner_centos7_expand
+  plan:
+  - aggregate:
+    - get: gpdb_src
+      passed: [compile_gpdb_centos7]
+    - get: bin_gpdb
+      passed: [compile_gpdb_centos7]
+      resource: bin_gpdb_centos7
+      trigger: [[ test_trigger ]]
+    - get: gpdb6-centos7-test
+  - task: ic_gpdb
+    file: gpdb_src/concourse/tasks/ic_gpdb.yml
+    image: gpdb6-centos7-test
+    params:
+      MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world
+      TEST_OS: centos
+      TEST_BINARY_SWAP: false
+      CONFIGURE_FLAGS: {{configure_flags}}
+      EXCLUDE_TESTS: instr_in_shmem_setup,instr_in_shmem_terminate,segspace_setup,segspace,segspace_cleanup,cursor,instr_in_shmem,instr_in_shmem_verify,gdd/prepare,gdd/dist-deadlock-01,gdd/dist-deadlock-04,gdd/dist-deadlock-05,gdd/dist-deadlock-06,gdd/dist-deadlock-07,gdd/dist-deadlock-102,gdd/dist-deadlock-103,gdd/dist-deadlock-104,gdd/dist-deadlock-106,gdd/non-lock-105,gdd/non-lock-107,gdd/avoid-qd-deadlock,gdd/prepare-for-local,gdd/local-deadlock-03,gdd/end,concurrent_update_distkeys,concurrent_update_epq
+      TEST_EXPAND: true
+
 {% endif %}
 {% if "sles" in os_types %}
 - name: icw_gporca_sles11
@@ -2242,6 +2265,7 @@ jobs:
       - icw_gporca_centos6
       - icw_gporca_centos7
       - icw_planner_centos7
+      - icw_planner_centos7_expand
  ##     - icw_gporca_sles11
  ##     - icw_gporca_sles12
  ##     - icw_planner_sles12
@@ -2300,6 +2324,7 @@ jobs:
       - compile_gpdb_centos7
       - icw_gporca_centos7
       - icw_planner_centos7
+      - icw_planner_centos7_expand
       - resource_group_centos7
  ##     - MADlib_Test_orca_centos7
  ##     - MADlib_Test_planner_centos7

concourse/scripts/common.bash

@@ -49,6 +49,86 @@ function install_and_configure_gpdb() {
   configure
 }
 
+# usage: gen_gpexpand_input <old_size> <new_size>
+function gen_gpexpand_input() {
+  local old="$1"
+  local new="$2"
+  local inputfile="/tmp/inputfile.${old}-${new}"
+  local i
+
+  for ((i=old; i<new; i++)); do
+    cat <<EOF
+$HOSTNAME:$HOSTNAME:$((25432+i*2)):$PWD/datadirs/expand/primary${i}:$((3+i*2)):${i}:p
+$HOSTNAME:$HOSTNAME:$((25433+i*2)):$PWD/datadirs/expand/mirror${i}:$((4+i*2)):${i}:m
+EOF
+  done | tee $inputfile
+
+  chmod a+r $inputfile
+}
+
+# extract PGOPTIONS from MAKE_TEST_COMMAND
+function get_pgoptions() {
+  local pgoptions
+
+  cat >/tmp/get_pgoptions.mk <<"EOF"
+$(info $(PGOPTIONS))
+EOF
+
+  pgoptions="$(eval make $MAKE_TEST_COMMAND -f /tmp/get_pgoptions.mk -nq 2>/dev/null)"
+  echo "$pgoptions"
+}
+
+# usage: expand_cluster <old_size> <new_size>
+function expand_cluster() {
+  local old="$1"
+  local new="$2"
+  local inputfile="/tmp/inputfile.${old}-${new}"
+  local pidfile="/tmp/postmaster.pid.${old}-${new}"
+  local dbname="postgres"
+  local pgoptions="$(get_pgoptions)"
+  local uncompleted
+  local partial
+
+  pushd gpdb_src/gpAux/gpdemo
+
+  gen_gpexpand_input "$old" "$new"
+
+  # Backup master pid, by checking it later we can know whether the cluster is
+  # restarted during the tests.
+  su gpadmin -c "head -n 1 $MASTER_DATA_DIRECTORY/postmaster.pid >$pidfile"
+  # begin expansion
+  su gpadmin -c "yes | PGOPTIONS='$pgoptions' gpexpand -s -i $inputfile"
+  # redistribute tables
+  su gpadmin -c "yes | PGOPTIONS='$pgoptions' gpexpand -s"
+  # check the result
+  uncompleted=$(su gpadmin -c "psql -Aqtd $dbname -c \"select count(*) from gpexpand.status_detail where status <> 'COMPLETED'\"")
+  # cleanup
+  su gpadmin -c "yes | PGOPTIONS='$pgoptions' gpexpand -s -c"
+
+  popd
+
+  if [ "$uncompleted" -ne 0 ]; then
+	  echo "error: some tables are not successfully expanded"
+	  return 1
+  fi
+
+  # double check gp_distribution_policy.numsegments
+  partial=$(su gpadmin -c "psql -Aqtd $dbname -c \"select count(*) from gp_distribution_policy where numsegments <> $new\"")
+  if [ "$partial" -ne 0 ]; then
+	  echo "error: not all the tables are expanded by gpexpand"
+	  return 1
+  fi
+
+  echo "all the tables are successfully expanded"
+  return 0
+}
+
+# usage: make_cluster [<demo_cluster_options>]
+#
+# demo_cluster_options are passed to `make create-demo-cluster` literally,
+# any options accepted by that command are acceptable here.
+#
+# e.g. make_cluster WITH_MIRRORS=false
 function make_cluster() {
   source /usr/local/greenplum-db-devel/greenplum_path.sh
   export BLDWRAP_POSTGRES_CONF_ADDONS=${BLDWRAP_POSTGRES_CONF_ADDONS}
@@ -57,7 +137,8 @@ function make_cluster() {
   export DEFAULT_QD_MAX_CONNECT=150
   export STATEMENT_MEM=250MB
   pushd gpdb_src/gpAux/gpdemo
-  su gpadmin -c "make create-demo-cluster"
+  export MASTER_DATA_DIRECTORY=`pwd`"/datadirs/qddir/demoDataDir-1"
+  su gpadmin -c "make create-demo-cluster $@"
   popd
 }
 

concourse/scripts/ic_gpdb.bash

@@ -78,9 +78,33 @@ function _main() {
 
     time install_and_configure_gpdb
     time setup_gpadmin_user
-    time make_cluster
+    # If $TEST_EXPAND is set, the job is to run ICW after expansion to see
+    # whether all the cases have been passed without restarting cluster.
+    # So it will create a cluster with two segments and execute gpexpand to
+    # expand the cluster to three segments
+    if [ "$TEST_EXPAND" = "true" ]; then
+        time make_cluster NUM_PRIMARY_MIRROR_PAIRS=2
+        time expand_cluster 2 3
+    else
+        time make_cluster
+    fi
     time gen_env
     time run_test
+    # If $TEST_EXPAND is set, the job is to run ICW after expansion to see
+    # whether all the cases have been passed without restarting cluster.
+    # Here is to check whether the cluster has been restarted by master pid.
+    # We wanna to be sure all the test cases have been passed after expansion
+    # without restarting the cluster. So any restarting is not expected.
+    if [ "$TEST_EXPAND" = "true" ]; then
+      OLD_MASTER_PID=`cat /tmp/postmaster.pid.2-3`
+      NEW_MASTER_PID=`head -n 1 gpdb_src/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/postmaster.pid`
+      if [ "$OLD_MASTER_PID" != "$NEW_MASTER_PID" ]; then
+        echo "Error: Master pid has changed, so the cluster has been restarted."
+        exit 1
+      fi
+      # Trigger gpexpand again after ICW.
+      time expand_cluster 3 4
+    fi
 
     if [ "${TEST_BINARY_SWAP}" == "true" ]; then
         time ./gpdb_src/concourse/scripts/test_binary_swap_gpdb.bash

contrib/pg_upgrade/Makefile

@@ -37,8 +37,15 @@ endif
 aomd_filehandler.c: % : $(top_srcdir)/src/backend/access/appendonly/%
 	rm -f $@ && $(LN_S) $< .
 
+# If TEST_EXPAND is set, the job should not contain any test cases with
+# restarting cluster operation, so this check should be disabled.
+ifndef TEST_EXPAND
 check: test_gpdb.sh all
 	bash $< -C -r -s -o $(top_builddir)/gpAux/gpdemo/datadirs/ -b $(DESTDIR)$(bindir)
+else
+check:
+	+@echo "pg_upgrade tests are skipped to prevent restarting the cluster"
+endif
 
 installcheck: test_gpdb.sh all
 	bash $< -o $(top_builddir)/gpAux/gpdemo/datadirs/ -b $(DESTDIR)$(bindir)