Remove ICW job icw_planner_centos7_online_expand (#7172)

This job is used to test online expand, it will create a cluster
with two segments, then expand to 3 and run all ICW to check
whether the cluster is OK after expansion.
As restart is forbidden in online expand test, so we exclude all
the case which contains restart opertaion. But if someone add a
new test with restart, the job may fail. Manual intervention to
exclude the test is needed.
So we move this job to our own dev pipeline to reduce the impact
on prod pipeline.

concourse/pipelines/gpdb_master-generated.yml

@@ -45,7 +45,6 @@ groups:
   - pg_upgrade
   - icw_gporca_centos7
   - icw_planner_centos7
-  - icw_planner_centos7_online_expand
   - icw_gporca_sles11
   - icw_gporca_sles12
   - icw_planner_sles12
@@ -126,7 +125,6 @@ groups:
   - pg_upgrade
   - icw_gporca_centos7
   - icw_planner_centos7
-  - icw_planner_centos7_online_expand
   - compile_gpdb_centos7
   - icw_gporca_sles11
   - icw_gporca_sles12
@@ -1095,27 +1093,6 @@ jobs:
       TEST_BINARY_SWAP: false
       CONFIGURE_FLAGS: {{configure_flags}}
 
-- name: icw_planner_centos7_online_expand
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [compile_gpdb_centos7]
-    - get: bin_gpdb
-      passed: [compile_gpdb_centos7]
-      resource: bin_gpdb_centos7
-      trigger: true
-    - get: gpdb6-centos7-test
-  - task: ic_gpdb
-    file: gpdb_src/concourse/tasks/ic_gpdb.yml
-    image: gpdb6-centos7-test
-    params:
-      MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world
-      TEST_OS: centos
-      TEST_BINARY_SWAP: false
-      CONFIGURE_FLAGS: {{configure_flags}}
-      EXCLUDE_TESTS: instr_in_shmem_setup,instr_in_shmem_terminate,segspace_setup,segspace,segspace_cleanup,cursor,instr_in_shmem,instr_in_shmem_verify,gdd/prepare,gdd/dist-deadlock-01,gdd/dist-deadlock-04,gdd/dist-deadlock-05,gdd/dist-deadlock-06,gdd/dist-deadlock-07,gdd/dist-deadlock-102,gdd/dist-deadlock-103,gdd/dist-deadlock-104,gdd/dist-deadlock-106,gdd/non-lock-105,gdd/non-lock-107,gdd/avoid-qd-deadlock,gdd/prepare-for-local,gdd/local-deadlock-03,gdd/end,concurrent_update_distkeys,concurrent_update_epq
-      ONLINE_EXPAND: true
-
 - name: icw_gporca_sles11
   plan:
   - aggregate:
@@ -2028,7 +2005,6 @@ jobs:
       - icw_gporca_centos6
       - icw_gporca_centos7
       - icw_planner_centos7
-      - icw_planner_centos7_online_expand
  ##     - icw_gporca_sles11
  ##     - icw_gporca_sles12
  ##     - icw_planner_sles12
@@ -2094,7 +2070,6 @@ jobs:
       - compile_gpdb_centos7
       - icw_gporca_centos7
       - icw_planner_centos7
-      - icw_planner_centos7_online_expand
       - resource_group_centos7
  ##     - MADlib_Test_orca_centos7
  ##     - MADlib_Test_planner_centos7

concourse/pipelines/templates/gpdb-tpl.yml

@@ -53,7 +53,6 @@ groups:
 {% if "centos7" in os_types %}
   - icw_gporca_centos7
   - icw_planner_centos7
-  - icw_planner_centos7_online_expand
 {% endif %}
 {% if "sles" in os_types %}
   - icw_gporca_sles11
@@ -180,7 +179,6 @@ groups:
 {% if "centos7" in os_types %}
   - icw_gporca_centos7
   - icw_planner_centos7
-  - icw_planner_centos7_online_expand
   - compile_gpdb_centos7
 {% endif %}
 {% if "sles" in os_types %}
@@ -1244,27 +1242,6 @@ jobs:
       TEST_BINARY_SWAP: false
       CONFIGURE_FLAGS: {{configure_flags}}
 
-- name: icw_planner_centos7_online_expand
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [compile_gpdb_centos7]
-    - get: bin_gpdb
-      passed: [compile_gpdb_centos7]
-      resource: bin_gpdb_centos7
-      trigger: [[ test_trigger ]]
-    - get: gpdb6-centos7-test
-  - task: ic_gpdb
-    file: gpdb_src/concourse/tasks/ic_gpdb.yml
-    image: gpdb6-centos7-test
-    params:
-      MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off' installcheck-world
-      TEST_OS: centos
-      TEST_BINARY_SWAP: false
-      CONFIGURE_FLAGS: {{configure_flags}}
-      EXCLUDE_TESTS: instr_in_shmem_setup,instr_in_shmem_terminate,segspace_setup,segspace,segspace_cleanup,cursor,instr_in_shmem,instr_in_shmem_verify,gdd/prepare,gdd/dist-deadlock-01,gdd/dist-deadlock-04,gdd/dist-deadlock-05,gdd/dist-deadlock-06,gdd/dist-deadlock-07,gdd/dist-deadlock-102,gdd/dist-deadlock-103,gdd/dist-deadlock-104,gdd/dist-deadlock-106,gdd/non-lock-105,gdd/non-lock-107,gdd/avoid-qd-deadlock,gdd/prepare-for-local,gdd/local-deadlock-03,gdd/end,concurrent_update_distkeys,concurrent_update_epq
-      ONLINE_EXPAND: true
-
 {% endif %}
 {% if "sles" in os_types %}
 - name: icw_gporca_sles11
@@ -2108,7 +2085,6 @@ jobs:
       - icw_gporca_centos6
       - icw_gporca_centos7
       - icw_planner_centos7
-      - icw_planner_centos7_online_expand
  ##     - icw_gporca_sles11
  ##     - icw_gporca_sles12
  ##     - icw_planner_sles12
@@ -2164,7 +2140,6 @@ jobs:
       - compile_gpdb_centos7
       - icw_gporca_centos7
       - icw_planner_centos7
-      - icw_planner_centos7_online_expand
       - resource_group_centos7
  ##     - MADlib_Test_orca_centos7
  ##     - MADlib_Test_planner_centos7

concourse/scripts/common.bash

@@ -49,17 +49,6 @@ function install_and_configure_gpdb() {
   configure
 }
 
-function gen_gpexpand_input() {
-  HOSTNAME=`hostname`
-  PWD=`pwd`
-  cat > input <<-EOF
-$HOSTNAME:$HOSTNAME:25436:$PWD/datadirs/expand/primary:7:2:p
-$HOSTNAME:$HOSTNAME:25437:$PWD/datadirs/expand/mirror:8:2:m
-EOF
-
-  chmod a+r input
-}
-
 function make_cluster() {
   source /usr/local/greenplum-db-devel/greenplum_path.sh
   export BLDWRAP_POSTGRES_CONF_ADDONS=${BLDWRAP_POSTGRES_CONF_ADDONS}
@@ -67,24 +56,8 @@ function make_cluster() {
   # require max_connections of at least 129.
   export DEFAULT_QD_MAX_CONNECT=150
   export STATEMENT_MEM=250MB
-  export PGPORT=15432
   pushd gpdb_src/gpAux/gpdemo
-  export MASTER_DATA_DIRECTORY=`pwd`"/datadirs/qddir/demoDataDir-1"
-  # If $ONLINE_EXPAND is set, the job is to run ICW after expansion to see
-  # whether all the cases have been passed without restarting cluster.
-  # So it will create a cluster with two segments and execute gpexpand to
-  # expand the cluster to three segments
-  if [ ! -z "$ONLINE_EXPAND" ]
-  then
-    su gpadmin -c "make create-demo-cluster NUM_PRIMARY_MIRROR_PAIRS=2"
-    # Backup master pid, to check it after the whole test, be sure there is
-    # no test case restarting the cluster
-    su gpadmin -c "head -n 1 $MASTER_DATA_DIRECTORY/postmaster.pid > master.pid.bk"
-    gen_gpexpand_input
-    su gpadmin -c "gpexpand -i input"
-  else
-    su gpadmin -c "make create-demo-cluster"
-  fi
+  su gpadmin -c "make create-demo-cluster"
   popd
 }
 

concourse/scripts/ic_gpdb.bash

@@ -81,21 +81,6 @@ function _main() {
     time make_cluster
     time gen_env
     time run_test
-    # If $ONLINE_EXPAND is set, the job is to run ICW after expansion to see
-    # whether all the cases have been passed without restarting cluster.
-    # Here is to check whether the cluster has been restarted by master pid.
-    # We wanna to be sure all the test cases have been passed after expansion
-    # without restarting the cluster. So any restarting is not expected.
-    if [ ! -z "$ONLINE_EXPAND" ]
-    then
-      OLD_MASTER_PID=`cat gpdb_src/gpAux/gpdemo/master.pid.bk`
-      NEW_MASTER_PID=`head -n 1 gpdb_src/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/postmaster.pid`
-      if [ "$OLD_MASTER_PID" != "$NEW_MASTER_PID" ]
-      then
-          echo "Master pid has changed, so the cluster has been restarted."
-          exit 1
-      fi
-    fi
 
     if [ "${TEST_BINARY_SWAP}" == "true" ]; then
         time ./gpdb_src/concourse/scripts/test_binary_swap_gpdb.bash

contrib/pg_upgrade/Makefile

@@ -37,14 +37,8 @@ endif
 aomd_filehandler.c: % : $(top_srcdir)/src/backend/access/appendonly/%
 	rm -f $@ && $(LN_S) $< .
 
-# If ONLINE_EXPAND is set, the job should not contain any test cases with
-# restarting cluster operation, so this check should be disabled.
-ifndef ONLINE_EXPAND
 check: test_gpdb.sh all
 	bash $< -C -r -s -o $(top_builddir)/gpAux/gpdemo/datadirs/ -b $(DESTDIR)$(bindir)
-else
-check:
-endif
 
 installcheck: test_gpdb.sh all
 	bash $< -o $(top_builddir)/gpAux/gpdemo/datadirs/ -b $(DESTDIR)$(bindir)

src/test/isolation2/Makefile

@@ -56,7 +56,7 @@ clean distclean:
 install: all gpdiff.pl gpstringsubs.pl
 
 installcheck: install
-	./pg_isolation2_regress --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --psqldir='$(PSQLDIR)' --inputdir=$(srcdir) --ao-dir=uao --schedule=$(srcdir)/isolation2_schedule --exclude-tests=$(EXCLUDE_TESTS)
+	./pg_isolation2_regress --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --psqldir='$(PSQLDIR)' --inputdir=$(srcdir) --ao-dir=uao --schedule=$(srcdir)/isolation2_schedule
 
 installcheck-resgroup: install
 	./pg_isolation2_regress --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_resgroup --psqldir='$(PSQLDIR)' --inputdir=$(srcdir) --dbname=isolation2resgrouptest --schedule=$(srcdir)/isolation2_resgroup_schedule

src/test/regress/GNUmakefile

@@ -193,7 +193,7 @@ check-tests: all tablespace-setup
 installcheck: installcheck-good
 
 installcheck-good: all twophase_pqexecparams hooktest query_info_hook_test
-	$(pg_regress_installcheck) $(REGRESS_OPTS)  --schedule=$(srcdir)/parallel_schedule --schedule=$(srcdir)/greenplum_schedule --exclude-tests=$(EXCLUDE_TESTS) --ao-dir=uao $(EXTRA_TESTS)
+	$(pg_regress_installcheck) $(REGRESS_OPTS)  --schedule=$(srcdir)/parallel_schedule --schedule=$(srcdir)/greenplum_schedule --ao-dir=uao $(EXTRA_TESTS)
 
 installcheck-parallel: all tablespace-setup
 	$(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule $(MAXCONNOPT) $(EXTRA_TESTS)