Remove gphostcache

Gphostcache has numerous issues, and has been a pain point for some
time. For this reason, we are removing it.

This commit moves the useful function of gphostcache - the hostname
deduping - to the gparray class, where a list of deduped hostnames is
returned from gpArray.get_hostlist().

There is a FIXME of correctly adding hostname to a newly added or
recovered mirror. The hostname resolution from address was incorrect and
faulty in its logic - an IP address never requires a hostname associated
with it. However, the hostname field in gp_segment_configuration should
be populated somehow - we recommend a "hostname" field addition to any
configuration files that require it. For now, we simple set the
"hostname" to "address" which ultimately delivers the same functionality
as the gphostcache implementation.
Co-authored-by: NTyler Ramer <tramer@vmware.com>
Co-authored-by: NJamie McAtamney <jmcatamney@vmware.com>

gpMgmt/bin/README.md

@@ -79,14 +79,6 @@ gparray.py
    +-  GpArray   - Configuration information for a Greenplum Database
          \-  Contains multiple SegmentPair objects
 
-gphostcache.py   
-  |
-  +- GpHost      - Information about a single Host
-  |    \- Contains all the Segment objects on the host
-  |
-  +- GpHostCache - Helps resolve interface->hostname lookups
-       \- Contains all the GpHosts referenced in a Greenplum Database
-
 gplog.py         - Utility functions to assist in Greenplum standard logging
 
 gpparseopts.py   - Wrapper around optparse library to aid in locating help files

gpMgmt/bin/gpconfig

@@ -19,7 +19,6 @@ import re
 try:
     from gppylib.gpparseopts import OptParser, OptChecker
     from gppylib.gparray import GpArray
-    from gppylib.gphostcache import *
     from gppylib.gplog import *
     from gppylib.commands.unix import *
     from gppylib.commands.gp import *
@@ -43,6 +42,7 @@ SAMEVALUE_GUCS = set(["gp_default_storage_options"])
 read_only_gucs = set()  # populated at runtime
 LOGGER = get_default_logger()
 setup_tool_logging(EXECNAME, getLocalHostname(), getUserName())
+gp_array = None
 
 
 def parseargs():
@@ -196,9 +196,9 @@ def confirm_user_wants_to_continue():
         raise Exception("User Aborted. Exiting.")
 
 
-def print_verbosely(options, normalized_hostname, hostname, directory):
+def print_verbosely(options, hostname, directory):
     if options.verbose:
-        msg = "normalized_host=%s host=%s dir=%s" % (normalized_hostname, hostname, directory)
+        msg = "host=%s dir=%s" % (hostname, directory)
         LOGGER.info(msg)
 
 
@@ -265,14 +265,14 @@ def _show_all_segment_values_always(options):
     return options.show == "port"
 
 # FIXME: add value to cmd_name.  We do not just do this due to encoding issues.
-def do_add_config_script(pool, hostname, segs, value, options):
+def do_add_config_script(pool, segs, value, options):
     for seg in segs:
-        print_verbosely(options, hostname, seg.hostname, seg.datadir)
-        cmd_name = "add %s parameter on host %s for seg %s" % (options.entry, hostname, seg.datadir)
+        print_verbosely(options, seg.hostname, seg.datadir)
+        cmd_name = "add %s parameter on host %s for seg %s" % (options.entry, seg.hostname, seg.datadir)
         cmd = GpConfigHelper(cmd_name, seg.datadir, options.entry, value=value,
                              removeParameter=options.remove,
                              ctxt=REMOTE,
-                             remoteHost=hostname)
+                             remoteHost=seg.hostname)
         pool.addCommand(cmd)
 
 
@@ -281,8 +281,6 @@ def do_change(options):
         enable_verbose_logging()
 
     try:
-        gp_array = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
-
         if not options.skipvalidation:
             conn = dbconn.connect(dbconn.DbURL(), True)
             guc = get_normal_guc(conn, options)
@@ -302,49 +300,24 @@ def do_change(options):
         raise Exception(msg)
 
     pool = WorkerPool()
-
-    host_cache = GpHostCache(gp_array, pool)
-    failed_pings = host_cache.ping_hosts(pool)
-
-    if failed_pings:
-        for i in failed_pings:
-            LOGGER.warning('unreachable host: ' + i.hostname)
-        confirm_user_wants_to_continue()
-
     failure = False
     try:
         # do the segments
         if not options.masteronly:
-            for host in host_cache.get_hosts():
+            if options.primaryvalue:
+                do_add_config_script(pool, [seg for seg in gp_array.getSegDbList() if seg.isSegmentPrimary()],
+                                     options.primaryvalue, options)
 
-                if options.primaryvalue:
-                    do_add_config_script(pool, host.hostname, [seg for seg in host.dbs if seg.isSegmentPrimary()],
-                                         options.primaryvalue, options)
+            if options.mirrorvalue:
+                do_add_config_script(pool, [seg for seg in gp_array.getSegDbList() if seg.isSegmentMirror()],
+                                     options.mirrorvalue, options)
 
-                if options.mirrorvalue:
-                    do_add_config_script(pool, host.hostname, [seg for seg in host.dbs if seg.isSegmentMirror()],
-                                         options.mirrorvalue, options)
+            if not options.primaryvalue and not options.mirrorvalue:
+                do_add_config_script(pool, [seg for seg in gp_array.getSegDbList()], options.value, options)
 
-                if not options.primaryvalue and not options.mirrorvalue:
-                    do_add_config_script(pool, host.hostname, host.dbs, options.value, options)
-
-        # do the master
+        # do the master and standby
         if options.mastervalue or options.remove:
-            print_verbosely(options, gp_array.master.hostname, gp_array.master.hostname, gp_array.master.datadir)
-            cmd_name = "remove %s parameter on master host %s" % (options.entry, gp_array.master.hostname)
-            cmd = GpConfigHelper(cmd_name, gp_array.master.datadir, options.entry, value=options.mastervalue,
-                                 removeParameter=options.remove, ctxt=REMOTE, remoteHost=gp_array.master.hostname)
-            pool.addCommand(cmd)
-
-            # do the standby master
-            if gp_array.standbyMaster:
-                print_verbosely(options, gp_array.standbyMaster.hostname, gp_array.standbyMaster.hostname,
-                                gp_array.standbyMaster.datadir)
-                cmd_name = "remove %s parameter on standbymaster host %s" % (options.entry, gp_array.standbyMaster.hostname)
-                cmd = GpConfigHelper(cmd_name, gp_array.standbyMaster.datadir, options.entry,
-                                     value=options.mastervalue, removeParameter=options.remove, ctxt=REMOTE,
-                                     remoteHost=gp_array.standbyMaster.hostname)
-                pool.addCommand(cmd)
+            do_add_config_script(pool, [seg for seg in [gp_array.master, gp_array.standbyMaster] if seg is not None], options.mastervalue, options)
 
         pool.join()
         items = pool.getCompletedItems()
@@ -462,16 +435,15 @@ def log_and_raise(err_str):
 
 
 def get_gucs_from_files(guc):
-    hosts, pool = _get_hosts()
+    pool = WorkerPool()
     gucs_found = []
-    for host in hosts:
-        for seg in host.dbs:
-            cmd_name = "get %s parameter on host %s" % (guc, host.hostname)
-            pool.addCommand(
-                GpConfigHelper(cmd_name, seg.datadir, guc,
-                               segInfo=seg, getParameter=True,
-                               ctxt=REMOTE,
-                               remoteHost=host.hostname))
+    for seg in gp_array.getDbList():
+        cmd_name = "get %s parameter on host %s" % (guc, seg.hostname)
+        pool.addCommand(
+            GpConfigHelper(cmd_name, seg.datadir, guc,
+                           segInfo=seg, getParameter=True,
+                           ctxt=REMOTE,
+                           remoteHost=seg.hostname))
 
     failedSegs = False
     pool.join()
@@ -496,18 +468,16 @@ def get_gucs_from_files(guc):
     return gucs_found
 
 
-def _get_hosts():
-    dburl = dbconn.DbURL()
-    gp_array = GpArray.initFromCatalog(dburl, utility=True)
-    pool = WorkerPool()
-    host_cache = GpHostCache(gp_array, pool, withMasters=True)
-    failed_pings = host_cache.ping_hosts(pool)
-    if failed_pings:
-        for i in failed_pings:
-            LOGGER.warning('unreachable host: ' + i.hostname)
-        confirm_user_wants_to_continue()  # todo test me
-    hosts = host_cache.get_hosts()
-    return hosts, pool
+def _set_gparray():
+    try:
+        global gp_array
+        gp_array = GpArray.initFromCatalog(dbconn.DbURL(), utility=True)
+    except DatabaseError as ex:
+        LOGGER.error(ex.__str__())
+        msg = 'Failed to connect to database, exiting without action. ' \
+              'This script can only be run when the database is up.'
+        LOGGER.error(msg)
+        raise Exception(msg)
 
 
 def do_show(options):
@@ -538,9 +508,10 @@ def check_gpexpand():
 
 def do_main():
     options = parseargs()
+    _set_gparray()
+
     if options.list:
         do_list(options.skipvalidation)
-
     elif options.show:
         do_show(options)
 

gpMgmt/bin/gpinitsystem

@@ -76,7 +76,6 @@ REMOTE_HOST_COUNT=0
 SINGLE_HOST_BATCH_LIMIT=4
 INPUT_CONFIG=""
 OUTPUT_CONFIG=""
-GPHOSTCACHELOOKUP=$WORKDIR/lib/gphostcachelookup.py
 STANDBY_RET_CODE=0
 
 #******************************************************************************
@@ -950,7 +949,7 @@ CREATE_QE_ARRAY () {
 	LAST_HOST=X
 	SEG_DIR_VECTOR=0
 	MIR_COUNT=-1
-	MASTER_HOST=`HOST_LOOKUP $MASTER_HOSTNAME`
+	MASTER_HOST=$MASTER_HOSTNAME
 	QD_PRIMARY_ARRAY=${MASTER_HOST}~${MASTER_HOSTNAME}~${MASTER_PORT}~${MASTER_DIRECTORY}/${SEG_PREFIX}${CONTENT_COUNT}~${DBID_COUNT}~${CONTENT_COUNT}~0
 	((DBID_COUNT=$DBID_COUNT+1));((CONTENT_COUNT=$CONTENT_COUNT+1))
 	for QE_PAIR in ${M_HOST_ARRAY[@]}
@@ -1283,10 +1282,7 @@ CREATE_QD_DB () {
 		BACKOUT_COMMAND "if [ -d $GP_DIR ]; then $EXPORT_LIB_PATH;export PGPORT=$GP_PORT; $PG_CTL -D $GP_DIR stop; fi"
 		BACKOUT_COMMAND "$ECHO \"Stopping Master instance\""
 		LOG_MSG "[INFO]:-Completed starting the Master in admin mode"
-		GP_HOSTNAME=`HOST_LOOKUP $GP_HOSTADDRESS`
-		if [ x"$GP_HOSTNAME" = x"__lookup_of_hostname_failed__" ]; then
-			ERROR_EXIT "[FATAL]:-Hostname lookup for host $GP_HOSTADDRESS failed."
-		fi
+		GP_HOSTNAME=$GP_HOSTADDRESS
 		PING_HOST $GP_HOSTNAME
 		RETVAL=$?
 		if [ $RETVAL -ne 0 ]; then
@@ -1298,17 +1294,6 @@ CREATE_QD_DB () {
 		LOG_MSG "[INFO]:-End Function $FUNCNAME"
 }
 
-# returns hostname for given address or "__lookup_of_hostname_failed__" if hostname not found
-HOST_LOOKUP() {
-	res=`echo $1 | $GPHOSTCACHELOOKUP`
-	err_index=`echo $res | awk '{print index($0,"__lookup_of_hostname_failed__")}'`
-	if [ $err_index -ne 0 ]; then
-		echo "__lookup_of_hostname_failed__"
-	else
-		echo $res
-	fi
-}
-
 UPDATE_GPCONFIG () {
 		LOG_MSG "[INFO]:-Start Function $FUNCNAME"
 		MASTER_PORT=$1
@@ -1343,10 +1328,7 @@ LOAD_QE_SYSTEM_DATA () {
 		do
 				SET_VAR $I
 				LOG_MSG "[INFO]:-Adding segment $GP_HOSTADDRESS to Master system tables"
-				GP_HOSTNAME=`HOST_LOOKUP $GP_HOSTADDRESS`
-				if [ x"$GP_HOSTNAME" = x"__lookup_of_hostname_failed__" ]; then
-					ERROR_EXIT "[FATAL]:-Hostname lookup for host $GP_HOSTADDRESS failed."
-				fi
+				GP_HOSTNAME=$GP_HOSTADDRESS
 				PING_HOST $GP_HOSTNAME
 				RETVAL=$?
 				if [ $RETVAL -ne 0 ]; then

gpMgmt/bin/gppylib/Makefile

@@ -9,10 +9,23 @@ $(recurse)
 
 PROGRAMS= gparray.py gpunit unit2
 
-DATA= __init__.py datetimeutils.py fault_injection.py gp_era.py gpcatalog.py \
-	gphostcache.py gplog.py gpparseopts.py gpresgroup.py gpsubprocess.py \
-	gpversion.py heapchecksum.py logfilter.py mainUtils.py parseutils.py \
-	pgconf.py userinput.py utils.py
+DATA= __init__.py \
+	datetimeutils.py \
+	fault_injection.py \
+	gp_era.py \
+	gpcatalog.py \
+	gplog.py \
+	gpparseopts.py \
+	gpresgroup.py \
+	gpsubprocess.py \
+	gpversion.py \
+	heapchecksum.py \
+	logfilter.py \
+	mainUtils.py \
+	parseutils.py \
+	pgconf.py \
+	userinput.py \
+	utils.py
 
 installdirs:
 	$(MKDIR_P) '$(DESTDIR)$(libdir)/python/gppylib'

gpMgmt/bin/gppylib/gparray.py

@@ -759,20 +759,6 @@ def createSegmentRowsFromSegmentList( newHostlist
 
     return rows
 
-#========================================================================
-
-# TODO: Destroy this  (MPP-7686)
-#   Now that "hostname" is a distinct field in gp_segment_configuration
-#   attempting to derive hostnames from interaface names should be eliminated.
-#
-def get_host_interface(full_hostname):
-    (host_part, inf_num) = ['-'.join(full_hostname.split('-')[:-1]),
-                            full_hostname.split('-')[-1]]
-    if host_part == '' or not inf_num.isdigit():
-        return (full_hostname, None)
-    else:
-        return (host_part, inf_num)
-
 
 # ============================================================================
 class GpArray:
@@ -1212,8 +1198,8 @@ class GpArray:
                 hosts.append(self.standbyMaster.hostname)
         for segPair in self.segmentPairs:
             hosts.extend(segPair.get_hosts())
-        # dedupe? segPair.get_hosts() doesn't promise to dedupe itself, and there might be more deduping to do
-        return hosts
+        # dedupe
+        return list(set(hosts))
 
     # --------------------------------------------------------------------
     def get_master_host_names(self):
@@ -1366,14 +1352,11 @@ class GpArray:
            This currently assumes that all segments are configured the same
            and gets the results only from the host of segment 0
 
-        NOTE 2:
-           The determination of hostname is based on faulty logic
         """
 
         primary_datadirs = []
 
         seg0_hostname = self.segmentPairs[0].primaryDB.getSegmentAddress()
-        (seg0_hostname, inf_num) = get_host_interface(seg0_hostname)
 
         for db in self.getDbList():
             if db.isSegmentPrimary(False) and db.getSegmentAddress().startswith(seg0_hostname):
@@ -1390,7 +1373,6 @@ class GpArray:
         mirror_datadirs = []
 
         seg0_hostname = self.segmentPairs[0].primaryDB.getSegmentAddress()
-        (seg0_hostname, inf_num) = get_host_interface(seg0_hostname)
 
         for db in self.getDbList():
             if db.isSegmentMirror(False) and db.getSegmentAddress().startswith(seg0_hostname):

gpMgmt/bin/gppylib/gphostcache.py

-#!/usr/bin/env python
-#
-# Copyright (c) Greenplum Inc 2008. All Rights Reserved.
-#
-'''
-Greenplum hostcache file facilities.
-
-This Module contains some helper functions for mapping network
-interface names used in gp_segment_configuration to a collapsed set
-of hostnames.
-
-example:  sdw1-1, sdw1-2, sdw1-3, and sdw1-4 are all located
-          on sdw1.
-
-The results of this collapsing will be stored in a file:
-~/.gphostcache with entries of the form:
-
-sdw1-1:sdw1
-sdw1-2:sdw1
-sdw1-3:sdw1
-
-A big complication here is that we want to group all of the
-segment databases for sdw1-1 thru sdw1-4 together but we can't
-use the name returned by `hostname` as its not guaranteed to
-have a trusted ssh environment setup for it.
-
-'''
-import os
-
-from gppylib.commands import base
-from gppylib.commands import unix
-from gppylib import gplog
-from gppylib.utils import readAllLinesFromFile
-FILEDIR=os.path.expanduser("~")
-FILENAME=".gphostcache"
-CACHEFILE=FILEDIR + "/" + FILENAME
-
-logger = gplog.get_default_logger()
-
-class GpHost:
-    def __init__(self, hostname):
-        self.hostname = hostname
-        self.dbs=[]
-
-    def addDB(self,db):
-        self.dbs.append(db)
-
-    def __str__(self):
-        dirlist=[]
-        for db in self.dbs:
-            dirlist.append(db.datadir)
-        return "Host %s has Datadirs: [%s]" % (self.hostname,','.join(dirlist))
-
-class GpInterfaceToHostNameCache:
-
-    def __init__(self, pool, interfacesToLookup, currentHostNameAnswersForInterfaces):
-        self.__hostCache={}         # interface -> hostname
-
-        # Read the .gphostcache file if it exists
-        if os.path.isfile(CACHEFILE):
-            try:
-                for line in readAllLinesFromFile(CACHEFILE, stripLines=True, skipEmptyLines=True):
-                    if line[0] == '#': # okay check because empty lines are skipped
-                        continue
-
-                    arr = line.split(':')
-                    if len(arr) == 2 and len(arr[0]) > 0 and len(arr[1]) > 0:
-                        (interface, hostname) = arr
-                        self.__hostCache[interface.strip()]=hostname.strip()
-            except Exception, e:
-                logger.warn("Error reading file '%s': %s" % (CACHEFILE, str(e)))
-
-        #
-        # check to see which values are inconsistent with the cache and need lookup again
-        #
-        inconsistent = []
-        for i in range(len(interfacesToLookup)):
-            interface = interfacesToLookup[i]
-            hostname = currentHostNameAnswersForInterfaces[i]
-
-            # If we don't have this mapping yet set it, otherwise we simply
-            # validate consistency.
-            if interface not in self.__hostCache:
-                self.__hostCache[interface] = hostname
-            elif hostname is None:
-                # external source did not have a tentative answer, the first
-                # case above should have fired for the first entry on this
-                # interface and will force us to lookup the hostname.
-                # Additional hits on the interface can be ignored.
-                pass
-            elif self.__hostCache[interface] is None:
-                self.__hostCache[interface] = hostname
-            elif self.__hostCache[interface] != hostname:
-                logger.warn("inconsistent hostname '%s' for interface '%s' and expected hostname '%s'" % \
-                            (self.__hostCache[interface], interface, hostname))
-                inconsistent.append(interface)
-
-        # Clear out any inconsistent hostnames to force a recheck.
-        for i in inconsistent:
-            self.__hostCache[i] = None
-
-        # Lookup any hostnames that we don't have answers for:
-        pending_cmds={}
-        for interface in self.__hostCache:
-            if self.__hostCache[interface] is None:
-                logger.debug("hostname lookup for %s" % interface)
-                cmd=unix.Hostname('host lookup', ctxt=base.REMOTE, remoteHost=interface)
-                pool.addCommand(cmd)
-                pending_cmds[interface] = cmd
-
-        # Fetch the results out of the WorkerPool
-        if len(pending_cmds) > 0:
-            pool.join()
-
-            for interface in pending_cmds:
-                cmd = pending_cmds[interface]
-
-                # Make sure the command completed successfully
-                if cmd.get_results().rc != 0:
-                    logger.warn("Failed to resolve hostname for %s" % interface)
-                    continue
-
-                self.__hostCache[interface] = cmd.get_hostname()
-
-            pool.empty_completed_items()
-
-        # Try to update the hostcache file if we executed any hostname commands
-        if len(pending_cmds) > 0:
-            try:
-                fp = open(CACHEFILE, 'w')
-
-                for interface in sorted(self.__hostCache.keys()):
-                    hostname = self.__hostCache[interface]
-
-                    # skip any dangling references we still have
-                    if not hostname:
-                        continue
-                    fp.write("%s:%s\n" % (interface, hostname))
-
-                fp.close()
-            except Exception, e:
-                logger.warn(str(e))
-                logger.warn("Failed to write file '%s'" % CACHEFILE)
-
-
-    #
-    # returns the cached host name for the interface
-    #
-    # should only be called for interfaces that were passed to the constructor
-    #
-    # Will return None if lookup of the hostname was not possible
-    #
-    def getHostName(self, interface):
-        return self.__hostCache[interface]
-
-
-class GpHostCache:
-
-    def __init__(self, gparray, pool, skiplist=[], withMasters=False, segs=None):
-        self.gparray=gparray
-        self.gphost_map={}        # hostname -> GpHost
-        self.segs = segs
-
-        # these are any db's that should be skipped.
-        skipmap={}
-        for db in skiplist:
-            skipmap[db.getSegmentDbId()]=db
-
-        # Go through the gparray and build list of interface and hostname that
-        #  will be used to build the GpInterfaceToHostNameCache
-        #
-        # As Greenplum 4.0 we have both interface and hostname information in
-        # the catalog, so the gparray should be able to supply all of the
-        # information.
-        #
-        # However if we have initialized from an old catalog, or from a flatfile
-        # then the interface->hostname mapping may not be available.  In this
-        # case we still want to do the full pass first  so that we don't lookup
-        # a given interface more than once.
-        interfaces = []
-        hostnames = []
-
-        # Get list of segment dbs, optionally including masters
-        if withMasters:
-            dblist = self.gparray.getDbList()
-        else:
-            dblist = self.gparray.getSegDbList()
-
-        if segs:
-            dblist = segs
-
-        # build the interface->host mapping
-        for db in dblist:
-            if db.getSegmentDbId() not in skipmap:
-                interfaces.append(db.getSegmentAddress())
-                hostnames.append(db.getSegmentHostName())
-        interfaceToHostMap = \
-            GpInterfaceToHostNameCache(pool, interfaces, hostnames)
-
-        # Build up the GpHosts using our interface->hostname lookup
-        for db in dblist:
-
-            # skip this dbid ?
-            if db.getSegmentDbId() in skipmap:
-                continue
-            interface = db.getSegmentAddress()
-            hostname  = interfaceToHostMap.getHostName(interface)
-
-            # If the db didn't have hostname already set, (it was loaded from
-            # an old catalog?) set it based on the hostname from the interface
-            # lookup.
-            if db.getSegmentHostName() is None:
-                db.setSegmentHostName(hostname)
-
-            if hostname not in self.gphost_map:
-                self.gphost_map[hostname] = GpHost(hostname)
-
-            self.gphost_map[hostname].addDB(db)
-
-
-    ######
-    def log_contents(self):
-        logger.debug("Construct host-->datadirs mapping:")
-
-        entries=[]
-        for key in self.gphost_map.keys():
-            gphost=self.gphost_map[key]
-            entries.append(gphost.__str__())
-
-        logger.debug('\n'.join(entries))
-
-    ######
-    def get_hostnames(self):
-        hosts=[]
-        for key in self.gphost_map.keys():
-            gphost=self.gphost_map[key]
-            hosts.append(gphost.hostname)
-        return hosts
-
-    ######
-    def get_hosts(self):
-        return self.gphost_map.values()
-
-
-    ######
-    def get_host(self,hostname):
-        if hostname in self.gphost_map:
-            return self.gphost_map[hostname]
-        else:
-            raise Exception("map does not contain host: %s" % hostname)
-
-    #####
-    def ping_hosts(self, pool):
-        '''
-        go through all of the gphosts and try and ping all of the hosts.
-        If any fail then go back to using the interface names for those
-        segments.
-
-        throws an Exception if still can't ping on the interface names.
-        '''
-        failed_segs=[]
-        for key in self.gphost_map.keys():
-            p = unix.Ping('ping', key)
-            pool.addCommand(p)
-
-        pool.join()
-        cmds=pool.getCompletedItems()
-
-        for cmd in cmds:
-            # Look for commands that failed to ping
-            if not cmd.was_successful() != 0:
-                hostname=cmd.hostToPing
-                logger.warning("Ping to host: '%s' FAILED" % hostname)
-                logger.debug("  ping details: %s" % cmd)
-
-                gphost=self.get_host(hostname)
-                dblist=gphost.dbs
-
-                alternateHost=None
-
-                for db in dblist:
-                    dbid = db.getSegmentDbId()
-                    address = db.getSegmentAddress()
-
-                    # It would be nice to handle these through a pool,
-                    # but it is both a little difficult and not the
-                    # expected case.
-                    pingCmd = unix.Ping("dbid: %d" % dbid, address)
-                    pingCmd.run()
-                    if pingCmd.get_results().rc == 0:
-                        alternateHost=address
-                        logger.warning("alternate host: '%s' => '%s'" %
-                                       (hostname, address))
-                        break
-                    else:
-                        logger.warning("Ping to host: '%s' FAILED" % hostname)
-                        logger.debug("  ping details: %s" % pingCmd)
-
-                if alternateHost:
-                    gphost.hostname=alternateHost
-                else:
-                    # no interface to reach any of the segments, append all
-                    # segments to the list of failed segments
-                    failed_segs.extend(dblist)
-
-                    # Removing the failed host from the cache.
-                    #
-                    # This seems a bit draconian, but that is what all callers
-                    # of this function seem to want.
-                    del self.gphost_map[hostname]
-
-        pool.empty_completed_items()
-
-        return failed_segs

gpMgmt/bin/gppylib/gpresgroup.py

@@ -8,7 +8,6 @@ from gppylib.commands.unix import *
 from gppylib.commands.gp import *
 from gppylib.gparray import GpArray
 from gppylib.gplog import get_default_logger
-from gppylib.gphostcache import *
 
 class GpResGroup(object):
 

gpMgmt/bin/gppylib/operations/startSegments.py

@@ -6,7 +6,6 @@ from gppylib.commands import unix
 from gppylib.commands import gp
 from gppylib.commands import base
 from gppylib.gparray import GpArray
-from gppylib import gphostcache
 from gppylib.commands.gp import SEGMENT_TIMEOUT_DEFAULT
 
 logger = get_default_logger()
@@ -104,22 +103,6 @@ class StartSegmentsOperation:
         for seg in segments:
             segmentsDbids.append(seg.getSegmentDbId())
 
-        # check on pingability
-        self.hostcache = gphostcache.GpHostCache(gpArray, self.__workerPool)
-        failedPings=self.hostcache.ping_hosts(self.__workerPool)
-        failedPingDbIds = {}
-        for segment in failedPings:
-            hostName = segment.getSegmentHostName()
-            logger.warning("Skipping startup of segdb on %s directory %s Ping Failed <<<<<<" % \
-                                        (hostName, segment.getSegmentDataDirectory()))
-            if segment.getSegmentDbId() in segmentsDbids:
-               result.addFailure(segment, 'Failed to Ping on host: %s' % hostName, gp.SEGSTART_ERROR_PING_FAILED )
-            failedPingDbIds[segment.getSegmentDbId()] = True
-
-        self.hostcache.log_contents()
-
-        segments = [seg for seg in segments if failedPingDbIds.get(seg.getSegmentDbId()) is None]
-
         # now do the start!
         numNodes = len(gpArray.getHostList())
         numWorkers = self.__workerPool.getNumWorkers()

gpMgmt/bin/gppylib/programs/clsAddMirrors.py

@@ -25,7 +25,6 @@ from gppylib.system.environment import GpMasterEnvironment
 from gppylib.parseutils import line_reader, check_values, canonicalize_address
 from gppylib.utils import writeLinesToFile, readAllLinesFromFile, TableLogger, \
     PathNormalizationException, normalizeAndValidateInputPath
-from gppylib.gphostcache import GpInterfaceToHostNameCache
 from gppylib.userinput import *
 from gppylib.mainUtils import ExceptionNoStackTraceNeeded
 
@@ -292,8 +291,6 @@ class GpAddMirrorsProgram:
                 rows.append(self._getParsedRow(filename, lineno, line))
 
         allAddresses = [row["address"] for row in rows]
-        interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, [None]*len(allAddresses))
-
         #
         # build up the output now
         #
@@ -308,9 +305,8 @@ class GpAddMirrorsProgram:
             contentId = int(row['contentId'])
             address = row['address']
             dataDir = normalizeAndValidateInputPath(row['dataDirectory'], "in config file", row['lineno'])
-            hostName = interfaceLookup.getHostName(address)
-            if hostName is None:
-                raise Exception("Segment Host Address %s is unreachable" % address)
+            # FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file
+            hostName = address
 
             primary = segsByContentId[contentId]
             if primary is None:

gpMgmt/bin/gppylib/programs/clsRecoverSegment.py

@@ -36,7 +36,6 @@ from gppylib.system import configurationInterface as configInterface
 from gppylib.system.environment import GpMasterEnvironment
 from gppylib.parseutils import line_reader, check_values, canonicalize_address
 from gppylib.utils import writeLinesToFile, normalizeAndValidateInputPath, TableLogger
-from gppylib.gphostcache import GpInterfaceToHostNameCache
 from gppylib.operations.utils import ParallelOperation
 from gppylib.operations.package import SyncPackages
 from gppylib.heapchecksum import HeapChecksum
@@ -216,7 +215,6 @@ class GpRecoverSegmentProgram:
                 rows.append(self._getParsedRow(filename, lineno, line))
 
         allAddresses = [row["newAddress"] for row in rows if "newAddress" in row]
-        interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, [None]*len(allAddresses))
 
         failedSegments = []
         failoverSegments = []
@@ -263,10 +261,9 @@ class GpRecoverSegmentProgram:
 
                 dataDirectory = normalizeAndValidateInputPath(row["newDataDirectory"], "config file",
                                                               row['lineno'])
-
-                hostName = interfaceLookup.getHostName(address)
-                if hostName is None:
-                    raise Exception('Unable to find host name for address %s from line:%s' % (address, row['lineno']))
+                # FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file
+                # FIXME: This appears identical to __getMirrorsToBuildFromConfigFilein clsAddMirrors
+                hostName = address
 
                 # now update values in failover segment
                 failoverSegment.setSegmentAddress(address)

gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py

@@ -8,7 +8,6 @@ import sys
 import tempfile
 
 from gppylib.gparray import Segment, GpArray, SegmentPair
-from gppylib.gphostcache import GpHost
 from gpconfig_modules.parse_guc_metadata import ParseGuc
 import errno
 from pg import DatabaseError
@@ -53,18 +52,14 @@ class GpConfig(GpTestCase):
         self.os_env["MASTER_DATA_DIRECTORY"] = self.temp_dir
         self.os_env["GPHOME"] = self.temp_dir
         self.gparray = self._create_gparray_with_2_primary_2_mirrors()
-        self.host_cache = Mock()
 
-        self.host = GpHost('localhost')
+        self.host = 'localhost'
         seg = SegmentPair()
         db = self.gparray.master
         seg.addPrimary(db)
         seg.datadir = self.gparray.master.datadir
         seg.hostname = 'localhost'
-        self.host.addDB(seg)
 
-        self.host_cache.get_hosts.return_value = [self.host]
-        self.host_cache.ping_hosts.return_value = []
 
         self.master_file = Mock(name='master')
         self.master_file.get_value.return_value = 'foo'
@@ -84,7 +79,6 @@ class GpConfig(GpTestCase):
             patch('gpconfig.dbconn.connect', return_value=self.conn),
             patch('gpconfig.dbconn.query', return_value=self.cursor),
             patch('gpconfig.dbconn.querySingleton', side_effect=singleton_side_effect),
-            patch('gpconfig.GpHostCache', return_value=self.host_cache),
             patch('gpconfig.GpArray.initFromCatalog', return_value=self.gparray),
             patch('gpconfig.WorkerPool', return_value=self.pool)
         ])
@@ -236,11 +230,9 @@ class GpConfig(GpTestCase):
         # mocked values in the files
         self.pool.getCompletedItems.return_value.append(seg_1)
 
-        self.host_cache.get_hosts.return_value.extend([self.host, self.host])
-
         self.subject.do_main()
 
-        self.assertEqual(self.pool.addCommand.call_count, 3)
+        self.assertEqual(self.pool.addCommand.call_count, 5)
         self.assertEqual(self.subject.LOGGER.error.call_count, 0)
         self.assertIn("WARNING: GUCS ARE OUT OF SYNC", mock_stdout.getvalue())
         self.assertIn("bar", mock_stdout.getvalue())
@@ -261,11 +253,9 @@ class GpConfig(GpTestCase):
         # mocked values in the files
         self.pool.getCompletedItems.return_value.append(seg_1)
 
-        self.host_cache.get_hosts.return_value.extend([self.host, self.host])
-
         self.subject.do_main()
 
-        self.assertEqual(self.pool.addCommand.call_count, 3)
+        self.assertEqual(self.pool.addCommand.call_count, 5)
         self.assertEqual(self.subject.LOGGER.error.call_count, 0)
         self.assertIn("WARNING: GUCS ARE OUT OF SYNC", mock_stdout.getvalue())
         self.assertIn("bar", mock_stdout.getvalue())
@@ -284,12 +274,12 @@ class GpConfig(GpTestCase):
         self.subject.do_main()
 
         self.subject.LOGGER.info.assert_called_with("completed successfully with parameters '-c my_property_name -v 100 -m 20'")
-        self.assertEqual(self.pool.addCommand.call_count, 2)
+        self.assertEqual(self.pool.addCommand.call_count, 5)
         segment_command = self.pool.addCommand.call_args_list[0][0][0]
         self.assertTrue("my_property_name" in segment_command.cmdStr)
         value = base64.urlsafe_b64encode(pickle.dumps("100"))
         self.assertTrue(value in segment_command.cmdStr)
-        master_command = self.pool.addCommand.call_args_list[1][0][0]
+        master_command = self.pool.addCommand.call_args_list[4][0][0]
         self.assertTrue("my_property_name" in master_command.cmdStr)
         value = base64.urlsafe_b64encode(pickle.dumps("20"))
         self.assertTrue(value in master_command.cmdStr)
@@ -327,7 +317,7 @@ class GpConfig(GpTestCase):
         self.subject.do_main()
 
         self.subject.LOGGER.info.assert_called_with("completed successfully with parameters '-c my_hidden_guc_name -v 100 --skipvalidation'")
-        self.assertEqual(self.pool.addCommand.call_count, 2)
+        self.assertEqual(self.pool.addCommand.call_count, 5)
         segment_command = self.pool.addCommand.call_args_list[0][0][0]
         self.assertTrue("my_hidden_guc_name" in segment_command.cmdStr)
         master_command = self.pool.addCommand.call_args_list[1][0][0]

gpMgmt/bin/gppylib/test/unit/test_unit_gphostcachelookup.py

-#!/usr/bin/env python
-#
-# Copyright (c) Greenplum Inc 2012. All Rights Reserved. 
-#
-
-""" Unittesting for gphostcachelookup module
-"""
-import os, sys
-import unittest
-from gppylib import gplog
-from gppylib.commands.base import Command, ExecutionError
-
-FILEDIR=os.path.expanduser("~")
-FILENAME=".gphostcache"
-CACHEFILE = os.path.join(FILEDIR, FILENAME)
-
-logger = gplog.get_unittest_logger()
-
-try:
-    if os.getenv('GPHOME') is '':
-        raise Exception("Environment variable GPHOME not set")
-except Exception, e:
-    sys.stderr.write("Exception found: %s \n" % str(e))
-
-GPHOSTLOOKUP = os.path.join(os.getenv('GPHOME'), 'bin/lib/', 'gphostcachelookup.py')
-
-class GpHostCacheLookupTestCase(unittest.TestCase):
-
-    LOOKUP_FAIL_MSG = '__lookup_of_hostname_failed__'
-
-    def test01_gphost_lookup(self):
-        """ Test gphostcachelookup for known interface (present in host cache)"""
-
-        self._remove_old_cache_entries()
-        test_interface = "localhost"
-        hostname = os.uname()[1]
-
-        result = self._run_gphost_lookup(test_interface)
-
-        self.assertEqual(hostname, result)
-
-
-    def test02_gphost_lookup(self):
-        """Test gphostcachelookup for unknown interface (not present in host cache)"""
-
-        test_interface = "foo"
-
-        result = self._run_gphost_lookup(test_interface)
-
-        self.assertIn(self.LOOKUP_FAIL_MSG, result)
-
-
-    def test03_gphost_lookup(self):
-        """Test gphostcachelookup for empty interface name """
-
-        test_interface = ""
-
-        result = self._run_gphost_lookup(test_interface)
-
-        self.assertIn(self.LOOKUP_FAIL_MSG, result)
-
-
-#-------------------------------- Non-test Helper -----------------------
-
-    def _run_gphost_lookup(self, test_interface):
-        """This is a helper function to simply run 'gphostcachelookup.py' for given interface"""
-
-        
-        cmd = Command("Running host lookup for given interface",
-                        "echo %s | python %s" % (test_interface, GPHOSTLOOKUP))
-
-        try:
-            cmd.run(validateAfter = True)
-            
-        except ExecutionError, e:
-            self.fail("ExecutionError %s" % str(e))
-
-        result = cmd.get_results().stdout.strip()
-        return result
-
-
-
-    def _remove_old_cache_entries(self):
-        """ This function will be used to remove already present 'localhost' entries from gphostcache file """
-
-        with open(CACHEFILE, "r") as fr:
-            lines = fr.readlines()
-
-        with open(CACHEFILE, "w") as fw:
-            for line in lines:
-                if "localhost" not in line:
-                    fw.write(line)
-
-
-#------------------------------- Mainline --------------------------------
-if __name__ == '__main__':
-    unittest.main()

gpMgmt/bin/gppylib/test/unit/test_unit_gpstop.py

@@ -52,7 +52,6 @@ class GpStop(GpTestCase):
             patch('gpstop.unix', return_value=self.mock_unix),
             patch('gpstop.GpEraFile', return_value=self.mock_gperafile),
             patch('gpstop.GpArray.initFromCatalog'),
-            patch('gpstop.gphostcache.unix.Ping'),
             patch('gpstop.RemoteOperation'),
             patch('gpstop.base.WorkerPool'),
             patch('gpstop.socket.gethostname'),

gpMgmt/bin/gpstart

@@ -18,7 +18,6 @@ try:
     from gppylib.gpparseopts import OptParser, OptChecker
     from gppylib.gparray import *
     from gppylib.gplog import get_default_logger, log_to_file_only
-    from gppylib import gphostcache
     from gppylib import userinput
     from gppylib.db import catalog
     from gppylib.commands import unix
@@ -88,7 +87,6 @@ class GpStart:
     def run(self):
         self._prepare()
 
-        # MPP-13700
         if self.masteronly:
             if os.getenv('GPSTART_INTERNAL_MASTER_ONLY'):
                 logger.info('Master-only start requested for management utilities.')

gpMgmt/bin/gpstop

@@ -21,7 +21,6 @@ try:
     from gppylib.db import dbconn
     from gppylib.db import catalog
     from gppylib.gparray import *
-    from gppylib import gphostcache
     from gppylib import userinput
     from gppylib import pgconf
     from gppylib.commands import unix
@@ -116,7 +115,6 @@ class GpStop:
         self.dburl = None
         self.conn = None
         self.gparray = None
-        self.hostcache = None
         self.gpversion = None
 
         logger.debug("Setting level of parallelism to: %d" % self.parallel)
@@ -255,11 +253,11 @@ class GpStop:
         """
 
         logger.info('Cleaning up leftover shared memory')
-        cluster_hosts = set([gphost for gphost in self.hostcache.get_hosts()])
+        cluster_hosts = self.gparray.get_hostlist()
 
         operations = []
         for gphost in cluster_hosts:
-            operations.append(RemoteOperation(CleanSharedMem(gphost.dbs), gphost.hostname))
+            operations.append(RemoteOperation(CleanSharedMem(self.gparray.getDbList()), gphost))
 
         operations.append(
             RemoteOperation(CleanSharedMem([self.gparray.master]), self.gparray.master.getSegmentHostName()))
@@ -554,15 +552,6 @@ class GpStop:
 
         logger.info("Targeting dbid %s for shutdown" % [seg.getSegmentDbId() for seg in segs])
 
-        self.hostcache = gphostcache.GpHostCache(self.gparray, self.pool, segs=segs)
-
-        failed_pings = self.hostcache.ping_hosts(self.pool)
-        for db in failed_pings:
-            logger.warning(
-                "Skipping startup of segdb on %s directory %s Ping Failed <<<<<<" % (db.hostname, db.datadir))
-            failed_seg_status.append(SegStopStatus(db, False, 'Failed to Ping on host: %s' % db.hostname))
-
-        self.hostcache.log_contents()
 
         if self.gparray.hasMirrors:
             # stop primaries

gpMgmt/bin/lib/Makefile

@@ -7,8 +7,14 @@ SUBDIRS= pexpect
 
 $(recurse)
 
-PROGRAMS= __init__.py gp_bash_functions.sh gp_bash_version.sh gpconfigurenewsegment \
-	gpcreateseg.sh gphostcachelookup.py gppinggpfdist.py gpstate.py multidd
+PROGRAMS= __init__.py \
+	gp_bash_functions.sh \
+	gp_bash_version.sh \
+	gpconfigurenewsegment \
+	gpcreateseg.sh \
+	gppinggpfdist.py \
+	gpstate.py \
+	multidd
 
 installdirs:
 	$(MKDIR_P) '$(DESTDIR)$(bindir)/lib'

gpMgmt/bin/lib/gphostcachelookup.py

-#!/usr/bin/env python
-'''
-gphostcachelookup.py -- look up the hostname for a list of interfaces
-
-Usage: gphostcachelookup.py interface-name
-Input is taken from stdin. Each line from stdin is considered as interface name.
-Output is the hostname, gets printed to stdout.
-
-'''
-
-import sys
-from gppylib.gphostcache import GpInterfaceToHostNameCache
-from gppylib.commands import base
-
-#-------------------------------------------------------------------------
-if __name__ == '__main__':
-
-    pool = base.WorkerPool(1)
-    retCode = 0
-
-    try:
-        interfaces = []
-        hostNames = []
-        for line in sys.stdin:
-            interfaces.append(line.strip())
-            hostNames.append(None)
-
-        lookup = GpInterfaceToHostNameCache(pool, interfaces, hostNames)
-
-        for interface in interfaces:
-            hostname = lookup.getHostName(interface)
-            if hostname is None:
-                sys.stdout.write("__lookup_of_hostname_failed__\n")
-            else:
-                sys.stdout.write(hostname)
-                sys.stdout.write("\n")
-    except Exception, e:
-        sys.stderr.write("Exception converting hostname from cache: %s" % e.__str__())
-        sys.stderr.write("\n")
-        retCode = 1
-    except:
-        sys.stderr.write("Exception found converting interface to hostname")
-        sys.stderr.write("\n")
-        retCode = 1
-    finally:
-        if pool:
-            pool.haltWork()
-        sys.exit(retCode)
-
-

gpMgmt/doc/gpstart_help

@@ -27,13 +27,6 @@ at once (the master and all of the segment instances). The gpstart utility
 handles the startup of the individual instances. Each instance is started 
 in parallel.
 
-The first time an administrator runs gpstart, the utility creates a hosts 
-cache file named .gphostcache in the user's home directory. Subsequently, 
-the utility uses this list of hosts to start the system more efficiently. 
-If new hosts are added to the system, you must manually remove this file 
-from the gpadmin user's home directory. The utility will create a new hosts 
-cache file at the next startup.
-
 Before you can start a Greenplum Database system, you must have initialized 
 the system using gpinitsystem first.