Allow export of Docker and sensors plugins stats to InfluxDB, StatsD... (issue #600)

NEWS

@@ -7,6 +7,7 @@ Version 2.5
 
 Enhancements and new features:
 
+    * Allow export of Docker and sensors plugins stats to InfluxDB, StatsD... (issue #600)
     * Server password configuration for the browser mode (issue #500)
     * Display an error if export is not used in the standalone/client mode (issue #614)  
 

glances/core/glances_stats.py

@@ -129,14 +129,14 @@ class GlancesStats(object):
                 # generate self._exports_list["xxx"] = ...
                 self._exports[export_name] = export_module.Export(args=args, config=self.config)
         # Log plugins list
-        logger.debug("Available exports modules list: {0}".format(self.getAllExports()))
+        logger.debug("Available exports modules list: {0}".format(self.getExportList()))
         return True
 
     def getAllPlugins(self):
         """Return the plugins list."""
         return [p for p in self._plugins]
 
-    def getAllExports(self):
+    def getExportList(self):
         """Return the exports modules list."""
         return [p for p in self._exports]
 
@@ -174,6 +174,13 @@ class GlancesStats(object):
         """Return all the stats (list)."""
         return [self._plugins[p].get_raw() for p in self._plugins]
 
+    def getAllExports(self):
+        """
+        Return all the stats to be exported (list).
+        Default behavor is to export all the stat
+        """
+        return [self._plugins[p].get_export() for p in self._plugins]
+
     def getAllAsDict(self):
         """Return all the stats (dict)."""
         # Python > 2.6

glances/exports/glances_csv.py

@@ -68,7 +68,7 @@ class Export(GlancesExport):
         csv_data = []
 
         # Get the stats
-        all_stats = stats.getAll()
+        all_stats = stats.getAllExports()
         plugins = stats.getAllPlugins()
 
         # Loop over available plugin
@@ -79,7 +79,7 @@ class Export(GlancesExport):
                         # First line: header
                         if self.first_line:
                             csv_header += ('{0}_{1}_{2}'.format(
-                                plugin, stat[stat['key']], item) for item in stat)
+                                plugin, self.get_item_key(stat), item) for item in stat)
                         # Others lines: stats
                         fieldvalues = stat.values()
                         csv_data += fieldvalues

glances/exports/glances_export.py

@@ -65,35 +65,84 @@ class GlancesExport(object):
                 'processcount',
                 'ip',
                 'system',
-                'uptime']
+                'uptime',
+                'sensors',
+                'docker']
+
+    def get_item_key(self, item):
+        """Return the value of the item 'key'"""
+        try:
+            ret = item[item['key']]
+        except KeyError:
+            logger.error("No 'key' available in {}".format(item))
+        if isinstance(ret, list):
+            return ret[0]
+        else:
+            return ret
 
     def update(self, stats):
         """Update stats to a server.
 
         The method builds two lists: names and values
         and calls the export method to export the stats.
+
+        Be aware that CSV export overwrite this class and use a specific one.
         """
         if not self.export_enable:
             return False
 
-        # Get the stats
-        all_stats = stats.getAll()
+        # Get all the stats & limits
+        all_stats = stats.getAllExports()
         all_limits = stats.getAllLimits()
+        # Get the plugins list
         plugins = stats.getAllPlugins()
 
         # Loop over available plugins
         for i, plugin in enumerate(plugins):
             if plugin in self.plugins_to_export():
-                if isinstance(all_stats[i], list):
-                    for item in all_stats[i]:
-                        item.update(all_limits[i])
-                        export_names = list('{0}.{1}'.format(item[item['key']], key)
-                                            for key in item.keys())
-                        export_values = list(item.values())
-                        self.export(plugin, export_names, export_values)
-                elif isinstance(all_stats[i], dict):
-                    export_names = list(all_stats[i].keys()) + list(all_limits[i].keys())
-                    export_values = list(all_stats[i].values()) + list(all_limits[i].values())
-                    self.export(plugin, export_names, export_values)
+                if isinstance(all_stats[i], dict):
+                    all_stats[i].update(all_limits[i])
+                elif isinstance(all_stats[i], list):
+                    all_stats[i] += all_limits[i]
+                else:
+                    continue
+                export_names, export_values = self.__build_export(all_stats[i])
+                self.export(plugin, export_names, export_values)
 
         return True
+
+    def __build_export(self, stats):
+        """Build the export lists"""
+        export_names = []
+        export_values = []
+
+        if isinstance(stats, dict):
+            # Stats is a dict
+            # Is there a key ?
+            if 'key' in list(stats.keys()):
+                pre_key = '{}.'.format(stats[stats['key']])
+            else:
+                pre_key = ''
+            # Walk through the dict
+            for key, value in stats.iteritems():
+                if isinstance(value, list):
+                    try:
+                        value = value[0]
+                    except IndexError:
+                        value = ''
+                if isinstance(value, dict):
+                    item_names, item_values = self.__build_export(value)
+                    item_names = [pre_key + key.lower() + str(i) for i in item_names]
+                    export_names += item_names
+                    export_values += item_values
+                else:
+                    export_names.append(pre_key + key.lower())
+                    export_values.append(value)
+        elif isinstance(stats, list):
+            # Stats is a list (of dict)
+            # Recursive loop through the list
+            for item in stats:
+                item_names, item_values = self.__build_export(item)
+                export_names += item_names
+                export_values += item_values
+        return export_names, export_values

glances/exports/glances_influxdb.py

@@ -129,3 +129,4 @@ class Export(GlancesExport):
             self.client.write_points(data)
         except Exception as e:
             logger.error("Can not export stats to InfluxDB (%s)" % e)
+        logger.debug("Export {} stats to InfluxDB".format(name))

glances/exports/glances_statsd.py

@@ -100,3 +100,4 @@ class Export(GlancesExport):
                 self.client.gauge(stat_name, stat_value)
             except Exception as e:
                 logger.error("Can not export stats to Statsd (%s)" % e)
+        logger.debug("Export {} stats to Statsd".format(name))

glances/outputs/glances_bottle.py

@@ -51,7 +51,7 @@ class GlancesBottle(object):
         self._app.install(EnableCors())
         # Password
         if args.password != '':
-            self._app.install(auth_basic(self.check_auth));
+            self._app.install(auth_basic(self.check_auth))
         # Define routes
         self._route()
 

glances/plugins/glances_docker.py

@@ -60,6 +60,22 @@ class Plugin(GlancesPlugin):
         # Init the Docker API
         self.docker_client = False
 
+    def get_key(self):
+        """Return the key of the list."""
+        return 'name'
+
+    def get_export(self):
+        """Overwrite the default export method
+        - Only exports containers
+        - The key is the first container name
+        """
+        ret = []
+        try:
+            ret = self.stats['containers']
+        except KeyError as e:
+            logger.debug("Docker export error {}".format(e))
+        return ret
+
     def connect(self, version=None):
         """Connect to the Docker server."""
         # Init connection to the Docker API
@@ -180,6 +196,12 @@ class Plugin(GlancesPlugin):
                 c['memory'] = self.get_docker_memory(c['Id'], all_stats)
                 # c['network'] = self.get_docker_network(c['Id'], all_stats)
 
+                # Export name (first name in the list, without the /)
+                c['name'] = c['Names'][0][1:]
+
+                # The key is the container name and not the Id
+                c['key'] = self.get_key()
+
         elif self.input_method == 'snmp':
             # Update stats using SNMP
             # Not available

glances/plugins/glances_plugin.py

@@ -223,6 +223,10 @@ class GlancesPlugin(object):
         """Return the stats object."""
         return self.stats
 
+    def get_export(self):
+        """Return the stats object to export."""
+        return self.get_raw()
+
     def get_stats(self):
         """Return the stats object in JSON format."""
         return json.dumps(self.stats)

glances/plugins/glances_sensors.py

@@ -146,7 +146,11 @@ class Plugin(GlancesPlugin):
         - Battery capacity: 'battery'
         """
         for i in stats:
+            # Set the sensors type
             i.update({'type': sensor_type})
+            # also add the key name
+            i.update({'key': self.get_key()})
+
         return stats
 
     def update_views(self):