Ship subprocess32 and replace subprocess with it in python code (#8658)

* Ship modified python module subprocess32 again

subprocess32 is preferred over subprocess according to python documentation.
In addition we long ago modified the code to use vfork() against fork() to
avoid some "Cannot allocate memory" kind of error (false alarm though - memory
is actually sufficient) on gpdb product environment that is usually with memory
overcommit disabled.  And we compiled and shipped it also but later it was just
compiled but not shipped somehow due to makefile change (maybe a regression).
Let's ship it again.

* Replace subprocess with our own subprocess32 in python code.

Cherry-picked 9c4a885b and
              da724e8d and
              a8090c13 and
              4354f28c

gpAux/gpdemo/gpsegwalrep.py

@@ -28,7 +28,10 @@ Assuming all of the above, you can just run the tool as so:
 import argparse
 import os
 import sys
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 import threading
 import datetime
 import time

gpMgmt/Makefile

@@ -30,6 +30,11 @@ install: generate_greenplum_path_file
 	if [ -e bin/ext/yaml ]; then \
 	    cp -rp bin/ext/yaml $(DESTDIR)$(prefix)/lib/python ; \
 	fi
+	if [ -e bin/ext/subprocess32.py ]; then \
+	    cp -p bin/ext/subprocess32.py $(DESTDIR)$(prefix)/lib/python ; \
+	    cp -p bin/ext/subprocess32-ChangeLog $(DESTDIR)$(prefix)/lib/python ; \
+	    cp -p bin/ext/_posixsubprocess.so $(DESTDIR)$(prefix)/lib/python ; \
+	fi
 
 clean distclean:
 	$(MAKE) -C bin $@

gpMgmt/bin/Makefile

@@ -123,6 +123,7 @@ subprocess32:
 	@if [ `uname -s` = 'Linux' ]; then \
 		  cd $(PYLIB_SRC)/subprocess32 && CC="$(CC)" python setup.py build; \
 		  cp -f $(PYLIB_SRC)/subprocess32/build/lib.*/* $(PYLIB_DIR)/;  \
+		  cp -f $(PYLIB_SRC)/subprocess32/ChangeLog $(PYLIB_DIR)/subprocess32-ChangeLog;  \
 	  fi
 
 #

gpMgmt/bin/gpload.py

@@ -49,7 +49,11 @@ except Exception, e:
     sys.exit(2)
 
 import hashlib
-import datetime,getpass,os,signal,socket,subprocess,threading,time,traceback,re
+import datetime,getpass,os,signal,socket,threading,time,traceback,re
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 import uuid
 
 from gppylib.gpversion import GpVersion

gpMgmt/bin/gpload_test/gpload/TEST.py

@@ -10,7 +10,10 @@ import socket
 import fileinput
 import platform
 import re
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 from pygresql import pg
 
 """

gpMgmt/bin/gpload_test/gpload2/TEST.py

@@ -9,7 +9,10 @@ import socket
 import fileinput
 import platform
 import re
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 from pygresql import pg
 
 def get_port_from_conf():

gpMgmt/bin/gpload_test/gpload2/TEST_REMOTE.py

@@ -9,7 +9,10 @@ import socket
 import fileinput
 import platform
 import re
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 from shutil import copyfile
 from pygresql import pg
 

gpMgmt/bin/gppylib/commands/base.py

@@ -21,7 +21,10 @@ from threading import Thread
 
 import os
 import signal
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 import sys
 import time
 

gpMgmt/bin/gppylib/test/regress/test_regress_gpssh.py

@@ -3,7 +3,10 @@
 import os, signal, time, re
 import unittest
 import psutil
-from subprocess import PIPE
+try:
+    from subprocess32 import PIPE
+except:
+    from subprocess import PIPE
 
 class GpsshTestCase(unittest.TestCase):
 

gpMgmt/bin/gppylib/test/unit/test_unit_gpinitsystem.py

@@ -2,7 +2,10 @@ import os
 from mock import *
 from gp_unittest import *
 from StringIO import StringIO
-from subprocess import Popen, PIPE
+try:
+    from subprocess32 import Popen, PIPE
+except:
+    from subprocess import Popen, PIPE
 
 class GpInitSystemTest(GpTestCase):
     def setUp(self):

gpMgmt/bin/gppylib/test/unit/test_unit_gpstop.py

@@ -15,6 +15,11 @@ from gppylib.commands.base import Command, WorkerPool
 from gppylib.commands.gp import GpSegStopCmd
 from gppylib.mainUtils import ProgramArgumentValidationException
 
+try:
+    import subprocess32
+    mock_subprocess_str='subprocess32.call'
+except:
+    mock_subprocess_str='subprocess.call'
 
 class GpStop(GpTestCase):
     def setUp(self):
@@ -450,7 +455,7 @@ class GpStopPrintProgressTestCase(unittest.TestCase):
 
 
 @patch('gppylib.db.dbconn.connect')
-@patch('subprocess.call')
+@patch(mock_subprocess_str)
 class GpStopSmartModeTestCase(unittest.TestCase):
     def setUp(self):
         gpstop.logger = Mock(logging.Logger)

gpMgmt/bin/gppylib/utils.py

 import shutil, filecmp,re
 import os, fcntl, select, getpass, socket
 import stat
-from subprocess import *
+try:
+    from subprocess32 import *
+except:
+    from subprocess import *
 from sys import *
 from xml.dom import minidom
 from xml.dom import Node

gpMgmt/bin/pythonSrc/subprocess32/ChangeLog

+-----------------
+2015-03-03 3.2.6.1.gp
+-----------------
+* Use vfork instead fork to execute subprocess to avoid potential
+* false-alarmed oom issue in Linux environments which are configured
+* with kernel parameter overcommit_memory as 2 (disabling memory overcommit).
+* Typically, greenplum production environmments configure this usually.
+
 -----------------
 2014-04-23 3.2.6
 -----------------

gpMgmt/bin/pythonSrc/subprocess32/PKG-INFO

 Metadata-Version: 1.0
 Name: subprocess32
-Version: 3.2.6
+Version: 3.2.6.1.gp
 Summary: Backport of the subprocess module from Python 3.2/3.3 for use on 2.x.
 Home-page: http://code.google.com/p/python-subprocess32/
 Author: Gregory P. Smith

gpMgmt/bin/pythonSrc/subprocess32/setup.py

@@ -14,7 +14,7 @@ def main():
 
     setup(
       name='subprocess32',
-      version='3.2.6',
+      version='3.2.6.1.gp',
       description='Backport of the subprocess module from Python 3.2/3.3 for use on 2.x.',
       long_description="""
 This is a backport of the subprocess standard library module from

gpMgmt/test/behave/mgmt_utils/steps/gpconfig_mgmt_utils.py

 from os import path
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 from gppylib.db import dbconn
 from gppylib.gparray import GpArray
 

gpMgmt/test/behave/mgmt_utils/steps/gpssh_exkeys_mgmt_utils.py

@@ -2,7 +2,10 @@ from os import path
 import os
 import shutil
 import socket
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 import sys
 import tempfile
 

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -15,7 +15,10 @@ import tarfile
 import tempfile
 import thread
 import json
-from subprocess import Popen, PIPE
+try:
+    from subprocess32 import Popen, PIPE
+except:
+    from subprocess import Popen, PIPE
 import commands
 import signal
 from collections import defaultdict

gpMgmt/test/behave_utils/cluster_expand.py

 import glob
 from datetime import datetime, timedelta
-from subprocess import Popen, PIPE
+try:
+    from subprocess32 import Popen, PIPE
+except:
+    from subprocess import Popen, PIPE
 from utils import run_gpcommand
 
 from gppylib.commands.base import Command

gpMgmt/test/behave_utils/utils.py

@@ -8,7 +8,10 @@ import stat
 import time
 import glob
 import shutil
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 import difflib
 
 import yaml

gpcontrib/gp_replica_check/gp_replica_check.py

@@ -30,7 +30,10 @@ gp_replica_check.py -d "mydb1,mydb2,..." -r "hash,bitmap,gist,..."
 
 import argparse
 import sys
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 import threading
 import Queue
 import pipes  # for shell-quoting, pipes.quote()

src/test/isolation2/sql_isolation_testcase.py

@@ -17,7 +17,10 @@ limitations under the License.
 
 import pygresql.pg
 import os
-import subprocess
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 import re
 import multiprocessing
 import tempfile

src/test/regress/mem_quota_util.py

@@ -9,7 +9,11 @@ mkpath = lambda *x: os.path.join(MYD, *x)
 #globals
 SAMPLE_QUERY="(select count(*) from (select o0.o_orderkey from (heap_orders o0 left outer join heap_orders o1 on o0.o_orderkey = o1.o_orderkey left outer join heap_orders o2 on o2.o_orderkey = o1.o_orderkey left outer join heap_orders o3 on o3.o_orderkey = o2.o_orderkey left outer join heap_orders o4 on o4.o_orderkey = o3.o_orderkey) order by o0.o_orderkey) as foo);"
 
-import subprocess, shutil, time, re
+import shutil, time, re
+try:
+    import subprocess32 as subprocess
+except:
+    import subprocess
 from optparse import OptionParser, OptionGroup
 
 try: