Implement Folder plugin (issue #721)

NEWS

@@ -7,6 +7,7 @@ Version 2.x
 
 Enhancements and new features:
 
+    * New folders' monitoring plugins (issue #721)
     * Add process summary min/max stats (issue #703)
     * Add timestamp to the CSV export module (issue #708)
     * [WebUI] add "pointer" cursor for sortable columns (issue #704 from @notFloran)

README.rst

@@ -49,6 +49,7 @@ Optional dependencies:
 - ``matplotlib`` (for graphical/chart support)
 - ``pika`` (for the RabbitMQ/ActiveMQ export module)
 - ``py-cpuinfo`` (for the Quicklook CPU info module)
+- ``scandir`` (for the Folders plugin) [Only for Python < 3.5]
 
 Installation
 ============
@@ -93,7 +94,7 @@ features (like the Web interface):
 
 .. code-block:: console
 
-    pip install bottle batinfo https://bitbucket.org/gleb_zhulik/py3sensors/get/tip.tar.gz zeroconf netifaces pymdstat influxdb potsdb statsd pystache docker-py pysnmp pika py-cpuinfo
+    pip install bottle batinfo https://bitbucket.org/gleb_zhulik/py3sensors/get/tip.tar.gz zeroconf netifaces pymdstat influxdb potsdb statsd pystache docker-py pysnmp pika py-cpuinfo scandir
 
 Install or upgrade Glances from the Git ``develop`` repository:
 

conf/glances.conf

 [quicklook]
+# Define CPU, MEM and SWAP thresholds in %
 cpu_careful=50
 cpu_warning=70
 cpu_critical=90
@@ -10,6 +11,7 @@ swap_warning=70
 swap_critical=90
 
 [cpu]
+# Define CPU thresholds in %
 # Default values if not defined: 50/70/90
 user_careful=50
 user_warning=70
@@ -28,6 +30,7 @@ steal_critical=90
 #steal_log=True
 
 [percpu]
+# Define CPU thresholds in %
 # Default values if not defined: 50/70/90
 user_careful=50
 user_warning=70
@@ -40,6 +43,7 @@ system_warning=70
 system_critical=90
 
 [load]
+# Define LOAD thresholds
 # Value * number of cores
 # Default values if not defined: 0.7/1.0/5.0 per number of cores
 # Source: http://blog.scoutapp.com/articles/2009/07/31/understanding-load-averages
@@ -50,14 +54,14 @@ critical=5.0
 #log=False
 
 [mem]
-# Default limits for free RAM memory in %
+# Define RAM thresholds in %
 # Default values if not defined: 50/70/90
 careful=50
 warning=70
 critical=90
 
 [memswap]
-# Default limits for free swap memory in %
+# Define SWAP thresholds in %
 # Default values if not defined: 50/70/90
 careful=50
 warning=70
@@ -85,7 +89,7 @@ critical=90
 #sda1_alias=IntDisk
 
 [fs]
-# Default limits for free filesytem space in %
+# Define filesystem space thresholds in %
 # Default values if not defined: 50/70/90
 # It is also possible to define per mount point value
 # Example: /_careful=40
@@ -95,18 +99,35 @@ critical=90
 # Allow additionnals files types (comma-separated FS type)
 #allow=zfs
 
+#[folders]
+# Define a folder list to monitor
+# The list is composed of items (list_#nb <= 10)
+# An item is defined by:
+# * path: absolute path
+# * careful: optional careful threshold (in MB)
+# * warning: optional warning threshold (in MB)
+# * critical: optional critical threshold (in MB)
+#folder_1_path=/tmp
+#folder_1_careful=2500
+#folder_1_warning=3000
+#folder_1_critical=3500
+#folder_2_path=/home/nicolargo/Videos
+#folder_2_warning=17000
+#folder_2_critical=20000
+#folder_3_path=/nonexisting
+
 [sensors]
-# Sensors core limits (in Celsius...)
+# Sensors core thresholds (in Celsius...)
 # Default values if not defined: 60/70/80
 temperature_core_careful=60
 temperature_core_warning=70
 temperature_core_critical=80
-# Temperatures in °C for hddtemp
+# Temperatures threshold in °C for hddtemp
 # Default values if not defined: 45/52/60
 temperature_hdd_careful=45
 temperature_hdd_warning=52
 temperature_hdd_critical=60
-# Battery % limits
+# Battery threshold in %
 battery_careful=80
 battery_warning=90
 battery_critical=95
@@ -117,7 +138,7 @@ battery_critical=95
 #core 1_alias=CPU Core 1
 
 [processlist]
-# Limit values for CPU/MEM per process in %
+# Define CPU/MEM (per process) thresholds in %
 # Default values if not defined: 50/70/90
 cpu_careful=50
 cpu_warning=70
@@ -176,7 +197,7 @@ mem_critical=90
 
 [influxdb]
 # Configuration file for the --export-influxdb option
-# https://influxdb.com/ 
+# https://influxdb.com/
 host=localhost
 port=8086
 user=root

docs/glances-doc.rst

@@ -159,6 +159,7 @@ Command-Line Options
   --disable-ip          disable IP module
   --disable-diskio      disable disk I/O module
   --disable-fs          disable file system module
+  --disable-folder      disable folders monitoring module
   --disable-sensors     disable sensors module
   --disable-hddtemp     disable hddtemp module
   --disable-raid        disable RAID module
@@ -247,7 +248,7 @@ The following commands (key pressed) are supported while in Glances:
 ``E``
     Erase current process filter
 ``f``
-    Show/hide file system stats
+    Show/hide file system and folder monitoring stats
 ``F``
     Switch between file system used and free space
 ``g``
@@ -573,6 +574,39 @@ following section in the configuration file:
     [fs]
     allow=zfs,misc
 
+Folders
+-------
+
+The folders plugin allows user, through the configuration file, to monitor size
+of a predefined folders' list.
+
+.. image:: images/folders.png
+
+Each item is defined by:
+
+* ``path``: absolute path to monitor.
+* ``warning``: optional careful threshold (in MB).
+* ``warning``: optional warning threshold (in MB)
+* ``critical``: optional critical threshold (in MB)
+
+Up to 10 items can be defined.
+
+For example, if you want to monitor the /tmp folder, the following definition
+should do the job (threshold careful if size is > 2.5 GB, warning if > 3 Gb and
+critical if > 3.5 GB):
+
+::
+
+    [folders]
+    folder_1_path=/tmp
+    folder_1_careful=2500
+    folder_1_warning=3000
+    folder_1_critical=3500
+
+In client/server mode, the list is defined on the server side.
+
+Note: do *NOT* define folder containing lot of files and subfolders.
+
 Sensors
 -------
 

glances/folder_list.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of Glances.
+#
+# Copyright (C) 2015 Nicolargo <nicolas@nicolargo.com>
+#
+# Glances is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Glances is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Manage the folder list."""
+
+import os
+
+from glances.compat import range, u
+from glances.logger import logger
+
+# Use the built-in version of scandir/walk if possible, otherwise
+# use the scandir module version
+
+scandir_tag = True
+try:
+    # For Python 3.5 or higher
+    from os import scandir
+except ImportError:
+    # For others...
+    try:
+        from scandir import scandir
+    except Exception, e:
+        scandir_tag = False
+
+
+class FolderList(object):
+
+    """This class describes the optional monitored folder list.
+
+    The folder list is a list of 'important' folder to monitor.
+
+    The list (Python list) is composed of items (Python dict).
+    An item is defined (dict keys):
+    * path: Path to the folder
+    * careful: optional careful threshold (in MB)
+    * warning: optional warning threshold (in MB)
+    * critical: optional critical threshold (in MB)
+    """
+
+    # Maximum number of items in the list
+    __folder_list_max_size = 10
+    # The folder list
+    __folder_list = []
+
+    def __init__(self, config):
+        """Init the folder list from the configuration file, if it exists."""
+        self.config = config
+
+        if self.config is not None and self.config.has_section('folders'):
+            if scandir_tag:
+                # Process monitoring list
+                logger.debug("Folder list configuration detected")
+                self.__set_folder_list('folders')
+            else:
+                logger.error('Scandir not found. Please use Python 3.5+ or install the scandir lib')
+        else:
+            self.__folder_list = []
+
+    def __set_folder_list(self, section):
+        """Init the monitored folder list.
+
+        The list is defined in the Glances configuration file.
+        """
+        for l in range(1, self.__folder_list_max_size + 1):
+            value = {}
+            key = 'folder_' + str(l) + '_'
+
+            # Path is mandatory
+            try:
+                value['path'] = self.config.get_value(section, key + 'path')
+            except Exception as e:
+                logger.error("Cannot read folder list: {0}".format(e))
+                continue
+            if value['path'] is None:
+                continue
+
+            # Optional conf keys
+            for i in ['careful', 'warning', 'critical']:
+                try:
+                    value[i] = self.config.get_value(section, key + i)
+                except:
+                    value[i] = None
+                    logger.debug("No {0} threshold for folder {1}".format(i, value["path"]))
+
+            # Add the item to the list
+            self.__folder_list.append(value)
+
+    def __str__(self):
+        return str(self.__folder_list)
+
+    def __repr__(self):
+        return self.__folder_list
+
+    def __getitem__(self, item):
+        return self.__folder_list[item]
+
+    def __len__(self):
+        return len(self.__folder_list)
+
+    def __get__(self, item, key):
+        """Meta function to return key value of item.
+
+        Return None if not defined or item > len(list)
+        """
+        if item < len(self.__folder_list):
+            try:
+                return self.__folder_list[item][key]
+            except Exception:
+                return None
+        else:
+            return None
+
+    def __folder_size(self, path):
+        """Return the size of the directory given by path
+
+        path: <string>"""
+
+        ret = 0
+        for f in scandir(path):
+            if f.is_dir() and (f.name != '.' or f.name != '..'):
+                ret += self.__folder_size(os.path.join(path, f.name))
+            else:
+                try:
+                    ret += f.stat().st_size
+                except OSError:
+                    pass
+
+        return ret
+
+    def update(self):
+        """Update the command result attributed."""
+        # Only continue if monitor list is not empty
+        if len(self.__folder_list) == 0:
+            return self.__folder_list
+
+        # Iter upon the folder list
+        for i in range(len(self.get())):
+            # Update folder size
+            try:
+                self.__folder_list[i]['size'] = self.__folder_size(self.path(i))
+            except Exception as e:
+                self.__folder_list[i]['size'] = None
+                logger.debug('Can get folder size ({0}). Error: {1}'.format(self.path(i), e))
+
+        return self.__folder_list
+
+    def get(self):
+        """Return the monitored list (list of dict)."""
+        return self.__folder_list
+
+    def set(self, newlist):
+        """Set the monitored list (list of dict)."""
+        self.__folder_list = newlist
+
+    def getAll(self):
+        # Deprecated: use get()
+        return self.get()
+
+    def setAll(self, newlist):
+        # Deprecated: use set()
+        self.set(newlist)
+
+    def path(self, item):
+        """Return the path of the item number (item)."""
+        return self.__get__(item, "path")
+
+    def careful(self, item):
+        """Return the careful threshold of the item number (item)."""
+        return self.__get__(item, "careful")
+
+    def warning(self, item):
+        """Return the warning threshold of the item number (item)."""
+        return self.__get__(item, "warning")
+
+    def critical(self, item):
+        """Return the critical threshold of the item number (item)."""
+        return self.__get__(item, "critical")

glances/main.py

@@ -118,6 +118,8 @@ Start the client browser (browser mode):\n\
                             dest='disable_diskio', help='disable disk I/O module')
         parser.add_argument('--disable-fs', action='store_true', default=False,
                             dest='disable_fs', help='disable filesystem module')
+        parser.add_argument('--disable-folder', action='store_true', default=False,
+                            dest='disable_folder', help='disable folder module')
         parser.add_argument('--disable-sensors', action='store_true', default=False,
                             dest='disable_sensors', help='disable sensors module')
         parser.add_argument('--disable-hddtemp', action='store_true', default=False,

glances/outputs/glances_curses.py

@@ -334,8 +334,9 @@ class _GlancesCurses(object):
             # 'F' > Switch between FS available and free space
             self.args.fs_free_space = not self.args.fs_free_space
         elif self.pressedkey == ord('f'):
-            # 'f' > Show/hide fs stats
+            # 'f' > Show/hide fs / folder stats
             self.args.disable_fs = not self.args.disable_fs
+            self.args.disable_folder = not self.args.disable_folder
         elif self.pressedkey == ord('g'):
             # 'g' > History
             self.history_tag = not self.history_tag
@@ -496,6 +497,8 @@ class _GlancesCurses(object):
             'diskio').get_stats_display(args=self.args)
         stats_fs = stats.get_plugin('fs').get_stats_display(
             args=self.args, max_width=plugin_max_width)
+        stats_folder = stats.get_plugin('folder').get_stats_display(
+            args=self.args, max_width=plugin_max_width)
         stats_raid = stats.get_plugin('raid').get_stats_display(
             args=self.args)
         stats_sensors = stats.get_plugin(
@@ -659,8 +662,11 @@ class _GlancesCurses(object):
         # Display left sidebar (NETWORK+DISKIO+FS+SENSORS+Current time)
         # ==================================================================
         self.init_column()
-        if not (self.args.disable_network and self.args.disable_diskio and
-                self.args.disable_fs and self.args.disable_raid and
+        if not (self.args.disable_network and
+                self.args.disable_diskio and
+                self.args.disable_fs and
+                self.args.disable_folder and
+                self.args.disable_raid and
                 self.args.disable_sensors) and not self.args.disable_left_sidebar:
             self.new_line()
             self.display_plugin(stats_network)
@@ -669,6 +675,8 @@ class _GlancesCurses(object):
             self.new_line()
             self.display_plugin(stats_fs)
             self.new_line()
+            self.display_plugin(stats_folder)
+            self.new_line()
             self.display_plugin(stats_raid)
             self.new_line()
             self.display_plugin(stats_sensors)

glances/plugins/glances_folder.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of Glances.
+#
+# Copyright (C) 2015 Nicolargo <nicolas@nicolargo.com>
+#
+# Glances is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Glances is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Folder plugin."""
+
+from glances.compat import u
+from glances.folder_list import FolderList as glancesFolderList
+from glances.plugins.glances_plugin import GlancesPlugin
+from glances.logger import logger
+
+
+class Plugin(GlancesPlugin):
+
+    """Glances folder plugin."""
+
+    def __init__(self, args=None):
+        """Init the plugin."""
+        super(Plugin, self).__init__(args=args)
+
+        # We want to display the stat in the curse interface
+        self.display_curse = True
+
+        # Init stats
+        self.glances_folders = None
+        self.stats = []
+
+    def load_limits(self, config):
+        """Load the foldered list from the config file, if it exists."""
+        self.glances_folders = glancesFolderList(config)
+
+    def update(self):
+        """Update the foldered list."""
+        if self.input_method == 'local':
+            # Folder list only available in a full Glances environment
+            # Check if the glances_folder instance is init
+            if self.glances_folders is None:
+                return self.stats
+
+            # Update the foldered list (result of command)
+            self.glances_folders.update()
+
+            # Put it on the stats var
+            self.stats = self.glances_folders.get()
+        else:
+            pass
+
+        return self.stats
+
+    def get_alert(self, stat):
+        """Manage limits of the folder list"""
+
+        if stat['size'] is None:
+            return 'DEFAULT'
+        else:
+            ret = 'OK'
+
+        if stat['critical'] is not None and stat['size'] > int(stat['critical']) * 1000000:
+            ret = 'CRITICAL'
+        elif stat['warning'] is not None and stat['size'] > int(stat['warning']) * 1000000:
+            ret = 'WARNING'
+        elif stat['careful'] is not None and stat['size'] > int(stat['careful']) * 1000000:
+            ret = 'CAREFUL'
+
+        return ret
+
+    def msg_curse(self, args=None):
+        """Return the dict to display in the curse interface."""
+        # Init the return message
+        ret = []
+
+        # Only process if stats exist and display plugin enable...
+        if not self.stats or args.disable_folder:
+            return ret
+
+        # Build the string message
+        # Header
+        msg = '{0}'.format('FOLDERS')
+        ret.append(self.curse_add_line(msg, "TITLE"))
+
+        # Data
+        for i in self.stats:
+            ret.append(self.curse_new_line())
+            if len(i['path']) > 15:
+                # Cut path if it is too long
+                path = '_' + i['path'][-15 + 1:]
+            else:
+                path = i['path']
+            msg = '{0:<16} '.format(path)
+            ret.append(self.curse_add_line(msg))
+            try:
+                msg = '{0:>6}'.format(self.auto_unit(i['size']))
+            except:
+                msg = '{0:>6}'.format('?')
+            ret.append(self.curse_add_line(msg, self.get_alert(i)))
+
+        return ret

glances/plugins/glances_fs.py

@@ -221,7 +221,7 @@ class Plugin(GlancesPlugin):
         msg = '{0:>7}'.format('Total')
         ret.append(self.curse_add_line(msg))
 
-        # Disk list (sorted by name)
+        # Filesystem list (sorted by name)
         for i in sorted(self.stats, key=operator.itemgetter(self.get_key())):
             # New line
             ret.append(self.curse_new_line())

glances/plugins/glances_sensors.py

@@ -202,11 +202,14 @@ class Plugin(GlancesPlugin):
                 else:
                     value = i['value']
                     unit = i['unit']
-                msg = '{0:>7.0f}{1}'.format(value, unit)
-                ret.append(self.curse_add_line(
-                    msg, self.get_views(item=i[self.get_key()],
-                                        key='value',
-                                        option='decoration')))
+                try:
+                    msg = '{0:>7.0f}{1}'.format(value, unit)
+                    ret.append(self.curse_add_line(
+                        msg, self.get_views(item=i[self.get_key()],
+                                            key='value',
+                                            option='decoration')))
+                except ValueError:
+                    pass
 
         return ret
 

man/glances.1

@@ -16,6 +16,8 @@ terminal or web interface.
 This tool is written in Python and uses the psutil library to fetch the statistical
 values from key elements, like CPU, load average, memory, network, disks, file
 systems, processes and so on.
+.PP
+The full documentation is available on the ReadTheDocs Web site (http://glances.readthedocs.org/en/latest/)
 .SH COMMAND-LINE OPTIONS
 The command-line options are the following:
 .TP
@@ -61,6 +63,9 @@ disable disk I/O module
 .B \-\-disable-fs
 disable file system module
 .TP
+.B \-\-disable-folder
+disable folders monitoring module
+.TP
 .B \-\-disable-sensors
 disable sensors module
 .TP

setup.py

@@ -55,7 +55,8 @@ setup(
         'DOCKER': ['docker-py'],
         'EXPORT': ['influxdb>=1.0.0', 'potsdb' 'statsd', 'pika'],
         'ACTION': ['pystache'],
-        'CPUINFO': ['py-cpuinfo']
+        'CPUINFO': ['py-cpuinfo'],
+        'FOLDERS': ['scandir']
     },
     packages=['glances'],
     include_package_data=True,