resgroup: detect gpdb cgroup comp dirs automatically.

Take cpu for example, by default we expect gpdb dir to locate at
cgroup/cpu/gpdb.  But we'll also check for the cgroup dirs of init
process (pid 1), e.g. cgroup/cpu/custom, then we'll look for gpdb dir at
cgroup/cpu/custom/gpdb, if it's found and has good permissions, it can
be used instead of the default one.

If any of the gpdb cgroup component dir can not be found under init
process' cgroup dirs or has bad permissions we'll fallback all the gpdb
cgroup component dirs to the default ones.

NOTE: This auto detection will look for memory & cpuset gpdb dirs even
on 5X.

(cherry picked from commit f3dc101a)

gpMgmt/bin/gpcheckresgroupimpl

@@ -9,29 +9,26 @@ import psutil
 from gppylib.commands import gp
 from gppylib import gpversion
 
+gpverstr = gp.GpVersion.local("", os.getenv("GPHOME"))
+gpver = gpversion.GpVersion(gpverstr)
+
 class dummy(object):
 
     def validate_all(self):
         exit("resource group is not supported on this platform")
 
 
-def detectCgroupMountPoint():
-    proc_mounts_path = "/proc/self/mounts"
-    if os.path.exists(proc_mounts_path):
-        with open(proc_mounts_path) as f:
-            for line in f:
-                mntent = line.split()
-                if mntent[2] != "cgroup": continue
-                mount_point = os.path.dirname(mntent[1])
-                return mount_point
-    return ""
-
 class cgroup(object):
 
-    mount_point = detectCgroupMountPoint()
-    tab = { 'r': os.R_OK, 'w': os.W_OK, 'x': os.X_OK, 'f': os.F_OK }
-    impl = "cgroup"
-    error_prefix = " is not properly configured: "
+    def __init__(self):
+        self.mount_point = self.detect_cgroup_mount_point()
+        self.tab = { 'r': os.R_OK, 'w': os.W_OK, 'x': os.X_OK, 'f': os.F_OK }
+        self.impl = "cgroup"
+        self.error_prefix = " is not properly configured: "
+
+        self.compdirs = self.detect_comp_dirs()
+        if not self.validate_comp_dirs():
+            self.compdirs = self.fallback_comp_dirs()
 
     def validate_all(self):
         """
@@ -44,44 +41,49 @@ class cgroup(object):
         if not self.mount_point:
             self.die("failed to detect cgroup mount point.")
 
-        self.validate_permission("cpu/gpdb/", "rwx")
-        self.validate_permission("cpu/gpdb/cgroup.procs", "rw")
-        self.validate_permission("cpu/gpdb/cpu.cfs_period_us", "rw")
-        self.validate_permission("cpu/gpdb/cpu.cfs_quota_us", "rw")
-        self.validate_permission("cpu/gpdb/cpu.shares", "rw")
+        if not self.compdirs:
+            self.die("failed to detect cgroup component dirs.")
 
-        self.validate_permission("cpuacct/gpdb/", "rwx")
-        self.validate_permission("cpuacct/gpdb/cgroup.procs", "rw")
-        self.validate_permission("cpuacct/gpdb/cpuacct.usage", "r")
-        self.validate_permission("cpuacct/gpdb/cpuacct.stat", "r")
+        self.validate_permission("cpu", "gpdb/", "rwx")
+        self.validate_permission("cpu", "gpdb/cgroup.procs", "rw")
+        self.validate_permission("cpu", "gpdb/cpu.cfs_period_us", "rw")
+        self.validate_permission("cpu", "gpdb/cpu.cfs_quota_us", "rw")
+        self.validate_permission("cpu", "gpdb/cpu.shares", "rw")
 
-        self.validate_permission("memory/memory.limit_in_bytes", "r")
+        self.validate_permission("cpuacct", "gpdb/", "rwx")
+        self.validate_permission("cpuacct", "gpdb/cgroup.procs", "rw")
+        self.validate_permission("cpuacct", "gpdb/cpuacct.usage", "r")
+        self.validate_permission("cpuacct", "gpdb/cpuacct.stat", "r")
+
+        self.validate_permission("memory", "memory.limit_in_bytes", "r")
 
         # resgroup memory auditor is introduced in 6.0 devel and backported
         # to 5.x branch since 5.6.1.  To provide backward compatibilities
         # memory permissions are only checked since 6.0.
-        gpverstr = gp.GpVersion.local("", os.getenv("GPHOME"))
-        gpver = gpversion.GpVersion(gpverstr)
         if gpver.version >= [6, 0, 0]:
-            self.validate_permission("memory/gpdb/", "rwx")
-            self.validate_permission("memory/gpdb/memory.limit_in_bytes", "rw")
-            self.validate_permission("memory/gpdb/memory.usage_in_bytes", "r")
+            self.validate_permission("memory", "gpdb/", "rwx")
+            self.validate_permission("memory", "gpdb/memory.limit_in_bytes", "rw")
+            self.validate_permission("memory", "gpdb/memory.usage_in_bytes", "r")
 
-            self.validate_permission("cpuset/gpdb/", "rwx")
-            self.validate_permission("cpuset/gpdb/cgroup.procs", "rw")
-            self.validate_permission("cpuset/gpdb/cpuset.cpus", "rw")
-            self.validate_permission("cpuset/gpdb/cpuset.mems", "rw")
+            self.validate_permission("cpuset", "gpdb/", "rwx")
+            self.validate_permission("cpuset", "gpdb/cgroup.procs", "rw")
+            self.validate_permission("cpuset", "gpdb/cpuset.cpus", "rw")
+            self.validate_permission("cpuset", "gpdb/cpuset.mems", "rw")
 
     def die(self, msg):
         exit(self.impl + self.error_prefix + msg)
 
-    def validate_permission(self, path, mode):
+    def validate_permission(self, comp, path, mode):
         """
         Validate permission on path.
         If path is a dir it must ends with '/'.
         """
         try:
-            fullpath = os.path.join(self.mount_point, path)
+            if comp not in self.compdirs:
+                self.die("can't find dir of cgroup component '%s'" % (comp))
+
+            compdir = self.compdirs[comp]
+            fullpath = os.path.join(self.mount_point, comp, compdir, path)
             pathtype = path[-1] == "/" and "directory" or "file"
             modebits = reduce(lambda x, y: x | y,
                               map(lambda x: self.tab[x], mode), 0)
@@ -96,6 +98,70 @@ class cgroup(object):
             self.die("can't check permission on %s '%s': %s" \
                      % (pathtype, fullpath, str(e)))
 
+    def validate_comp_dirs(self):
+        """
+        Validate existance of cgroup component dirs.
+
+        Return True if all the components dir exist and have good permission,
+        otherwise return False.
+        """
+
+        comps = ['cpu', 'cpuacct']
+        if gpver.version >= [6, 0, 0]:
+            comps.extend(['cpuset', 'memory'])
+
+        for comp in comps:
+            if comp not in self.compdirs:
+                return False
+
+            compdir = self.compdirs[comp]
+            fullpath = os.path.join(self.mount_point, comp, compdir, 'gpdb')
+
+            if not os.access(fullpath, os.R_OK | os.W_OK | os.X_OK):
+                return False
+
+        return True
+
+    def detect_cgroup_mount_point(self):
+        proc_mounts_path = "/proc/self/mounts"
+        if os.path.exists(proc_mounts_path):
+            with open(proc_mounts_path) as f:
+                for line in f:
+                    mntent = line.split()
+                    if mntent[2] != "cgroup": continue
+                    mount_point = os.path.dirname(mntent[1])
+                    return mount_point
+        return ""
+
+    def detect_comp_dirs(self):
+        compdirs = {}
+        path = "/proc/1/cgroup"
+
+        if not os.path.exists(path):
+            return compdirs
+
+        for line in open(path):
+            line = line.strip()
+            compid, compnames, comppath = line.split(":")
+            if not compnames or '=' in compnames:
+                continue
+            for compname in compnames.split(','):
+                compdirs[compname] = comppath.strip(os.path.sep)
+
+        return compdirs
+
+    def required_comps(self):
+        comps = ['cpu', 'cpuacct']
+        if gpver.version >= [6, 0, 0]:
+            comps.extend(['cpuset', 'memory'])
+        return comps
+
+    def fallback_comp_dirs(self):
+        compdirs = {}
+        for comp in self.required_comps():
+            compdirs[comp] = ''
+        return compdirs
+
 if __name__ == '__main__':
     if sys.platform.startswith('linux'):
         cgroup().validate_all()

gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckresgroupimpl.py

@@ -16,13 +16,12 @@ import gpcheckresgroupimpl
 
 from gppylib.commands import gp
 from gppylib import gpversion
+
 gpverstr = gp.GpVersion.local("", os.getenv("GPHOME"))
 gpver = gpversion.GpVersion(gpverstr)
 
 @unittest.skipUnless(sys.platform.startswith("linux"), "requires linux")
 class GpCheckResGroupImplCGroup(unittest.TestCase):
-    cgroup_mntpnt = None
-    cgroup_default_mntpnt = gpcheckresgroupimpl.detectCgroupMountPoint()
 
     def setUp(self):
         self.cgroup_mntpnt = tempfile.mkdtemp(prefix='fake-cgroup-mnt-')
@@ -35,6 +34,9 @@ class GpCheckResGroupImplCGroup(unittest.TestCase):
         self.cgroup = gpcheckresgroupimpl.cgroup()
         self.cgroup.mount_point = self.cgroup_mntpnt
         self.cgroup.die = self.mock_cgroup_die
+        self.cgroup.compdirs = self.cgroup.fallback_comp_dirs()
+
+        self.cgroup_default_mntpnt = self.cgroup.detect_cgroup_mount_point()
 
         os.mkdir(os.path.join(self.cgroup_mntpnt, "cpu", "gpdb"), 0700)
         self.touch(os.path.join(self.cgroup_mntpnt, "cpu", "gpdb", "cgroup.procs"), 0600)
@@ -73,9 +75,84 @@ class GpCheckResGroupImplCGroup(unittest.TestCase):
             pass
         os.chmod(path, mode)
 
+    def test_comp_lists(self):
+        # this looks like redundant as it's just a copy of required_comps(),
+        # however it is necessary to verify this unit test is up-to-date.
+        comps = ['cpu', 'cpuacct']
+        if gpver.version >= [6, 0, 0]:
+            comps.extend(['cpuset', 'memory'])
+        self.assertEqual(self.cgroup.required_comps(), comps)
+
+    def test_comp_dirs_validation(self):
+        self.assertTrue(self.cgroup.validate_comp_dirs())
+
+    def test_comp_dirs_validation_when_cpu_gpdb_dir_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "cpu", "gpdb"), 0100)
+        self.assertFalse(self.cgroup.validate_comp_dirs())
+        os.chmod(os.path.join(self.cgroup_mntpnt, "cpu", "gpdb"), 0700)
+
+    def test_comp_dirs_validation_when_cpu_gpdb_dir_missing(self):
+        shutil.rmtree(os.path.join(self.cgroup_mntpnt, "cpu", "gpdb"))
+        self.assertFalse(self.cgroup.validate_comp_dirs())
+
+    def test_comp_dirs_validation_when_cpuacct_gpdb_dir_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "cpuacct", "gpdb"), 0100)
+        self.assertFalse(self.cgroup.validate_comp_dirs())
+        os.chmod(os.path.join(self.cgroup_mntpnt, "cpuacct", "gpdb"), 0700)
+
+    def test_comp_dirs_validation_when_cpuacct_gpdb_dir_missing(self):
+        shutil.rmtree(os.path.join(self.cgroup_mntpnt, "cpuacct", "gpdb"))
+        self.assertFalse(self.cgroup.validate_comp_dirs())
+
+    def test_comp_dirs_validation_when_cpuset_gpdb_dir_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "cpuset", "gpdb"), 0100)
+        if gpver.version >= [6, 0, 0]:
+            self.assertFalse(self.cgroup.validate_comp_dirs())
+        else:
+            self.assertTrue(self.cgroup.validate_comp_dirs())
+        os.chmod(os.path.join(self.cgroup_mntpnt, "cpuset", "gpdb"), 0700)
+
+    def test_comp_dirs_validation_when_cpuset_gpdb_dir_missing(self):
+        shutil.rmtree(os.path.join(self.cgroup_mntpnt, "cpuset", "gpdb"))
+        if gpver.version >= [6, 0, 0]:
+            self.assertFalse(self.cgroup.validate_comp_dirs())
+        else:
+            self.assertTrue(self.cgroup.validate_comp_dirs())
+
+    def test_comp_dirs_validation_when_memory_gpdb_dir_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "memory", "gpdb"), 0100)
+        if gpver.version >= [6, 0, 0]:
+            self.assertFalse(self.cgroup.validate_comp_dirs())
+        else:
+            self.assertTrue(self.cgroup.validate_comp_dirs())
+        os.chmod(os.path.join(self.cgroup_mntpnt, "memory", "gpdb"), 0700)
+
+    def test_comp_dirs_validation_when_memory_gpdb_dir_missing(self):
+        shutil.rmtree(os.path.join(self.cgroup_mntpnt, "memory", "gpdb"))
+        if gpver.version >= [6, 0, 0]:
+            self.assertFalse(self.cgroup.validate_comp_dirs())
+        else:
+            self.assertTrue(self.cgroup.validate_comp_dirs())
+
     def test_proper_setup(self):
         self.cgroup.validate_all()
 
+    def test_proper_setup_with_non_default_cgroup_comp_dirs(self):
+        # set comp dir to comp.dir
+        compdirs = self.cgroup.compdirs
+        self.cgroup.compdirs = {}
+        for comp in compdirs.keys():
+            self.cgroup.compdirs[comp] = comp + '.dir'
+        # move /sys/fs/cgroup/comp to /sys/fs/cgroup/comp/comp.dir
+        for comp in self.cgroup.compdirs.keys():
+            compdir = self.cgroup.compdirs[comp]
+            olddir = os.path.join(self.cgroup_mntpnt, comp)
+            tmpdir = os.path.join(self.cgroup_mntpnt, compdir)
+            shutil.move(olddir, tmpdir)
+            os.mkdir(olddir, 0700)
+            shutil.move(tmpdir, olddir)
+        self.cgroup.validate_all()
+
     def test_when_cpu_gpdb_dir_missing(self):
         shutil.rmtree(os.path.join(self.cgroup_mntpnt, "cpu", "gpdb"))
         with self.assertRaisesRegexp(AssertionError, "directory '.*/cpu/gpdb/' does not exist"):

src/backend/utils/resgroup/resgroup-ops-dummy.c

@@ -112,10 +112,10 @@ ResGroupOps_AssignGroup(Oid group, ResGroupCaps *caps, int pid)
  * immediately.
  *
  * On success it return a fd to the OS group, pass it to
- * ResGroupOps_UnLockGroup() to unblock it.
+ * ResGroupOps_UnLockGroup() to unlock it.
  */
 int
-ResGroupOps_LockGroup(Oid group, const char *comp, bool block)
+ResGroupOps_LockGroup(Oid group, ResGroupCompType comp, bool block)
 {
 	unsupported_system();
 	return -1;

src/backend/utils/resgroup/resgroup.c

@@ -3792,6 +3792,7 @@ groupMemOnAlterForCgroup(Oid groupId, ResGroupData *group)
 static void
 groupApplyCgroupMemInc(ResGroupData *group)
 {
+	ResGroupCompType comp = RESGROUP_COMP_TYPE_MEMORY;
 	int32 memory_limit;
 	int32 memory_inc;
 	int fd;
@@ -3804,7 +3805,7 @@ groupApplyCgroupMemInc(ResGroupData *group)
 	if (memory_inc <= 0)
 		return;
 
-	fd = ResGroupOps_LockGroup(group->groupId, "memory", true);
+	fd = ResGroupOps_LockGroup(group->groupId, comp, true);
 	memory_limit = ResGroupOps_GetMemoryLimit(group->groupId);
 	ResGroupOps_SetMemoryLimitByValue(group->groupId, memory_limit + memory_inc);
 	ResGroupOps_UnLockGroup(group->groupId, fd);
@@ -3820,6 +3821,7 @@ groupApplyCgroupMemInc(ResGroupData *group)
 static void
 groupApplyCgroupMemDec(ResGroupData *group)
 {
+	ResGroupCompType comp = RESGROUP_COMP_TYPE_MEMORY;
 	int32 memory_limit;
 	int32 memory_dec;
 	int fd;
@@ -3827,7 +3829,7 @@ groupApplyCgroupMemDec(ResGroupData *group)
 	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
 	Assert(group->memGap > 0);
 
-	fd = ResGroupOps_LockGroup(group->groupId, "memory", true);
+	fd = ResGroupOps_LockGroup(group->groupId, comp, true);
 	memory_limit = ResGroupOps_GetMemoryLimit(group->groupId);
 	Assert(memory_limit > group->memGap);
 

src/include/utils/resgroup-ops.h

@@ -14,6 +14,22 @@
 #ifndef RES_GROUP_OPS_H
 #define RES_GROUP_OPS_H
 
+/*
+ * Resource Group underlying component types.
+ */
+typedef enum
+{
+	RESGROUP_COMP_TYPE_FIRST			= 0,
+	RESGROUP_COMP_TYPE_UNKNOWN			= -1,
+
+	RESGROUP_COMP_TYPE_CPU,
+	RESGROUP_COMP_TYPE_CPUACCT,
+	RESGROUP_COMP_TYPE_MEMORY,
+	RESGROUP_COMP_TYPE_CPUSET,
+
+	RESGROUP_COMP_TYPE_COUNT,
+} ResGroupCompType;
+
 #define RESGROUP_ROOT_ID (InvalidOid)
 /*
  * If group id is RESGROUP_COMPROOT_ID, it will build the root path of comp,
@@ -32,11 +48,12 @@
  * If cpu_rate_limit is set to this value, it means this feature is disabled
  */
 #define CPU_RATE_LIMIT_DISABLED (-1)
+
 /*
  * Interfaces for OS dependent operations
  */
 
-extern const char * ResGroupOps_Name(void);
+extern const char *ResGroupOps_Name(void);
 extern bool ResGroupOps_Probe(void);
 extern void ResGroupOps_Bless(void);
 extern void ResGroupOps_Init(void);
@@ -44,7 +61,7 @@ extern void ResGroupOps_AdjustGUCs(void);
 extern void ResGroupOps_CreateGroup(Oid group);
 extern void ResGroupOps_DestroyGroup(Oid group, bool migrate);
 extern void ResGroupOps_AssignGroup(Oid group, ResGroupCaps *caps, int pid);
-extern int ResGroupOps_LockGroup(Oid group, const char *comp, bool block);
+extern int ResGroupOps_LockGroup(Oid group, ResGroupCompType comp, bool block);
 extern void ResGroupOps_UnLockGroup(Oid group, int fd);
 extern void ResGroupOps_SetCpuRateLimit(Oid group, int cpu_rate_limit);
 extern void ResGroupOps_SetMemoryLimit(Oid group, int memory_limit);