Add a connector to ElasticSearch (welcome to Kibana dashboard) #311

NEWS

@@ -7,6 +7,7 @@ Version 2.6
 
 Enhancements and new features:
 
+    * Add a connector to ElasticSearch (welcome to Kibana dashboard) (issue #311)
     * New folders' monitoring plugins (issue #721)
     * Add an option to disable top menu (issue #766)
     * Add IOps in the DiskIO plugin (issue #763)

conf/glances.conf

@@ -223,6 +223,14 @@ host=localhost
 port=8125
 #prefix=glances
 
+[elasticsearch]
+# Configuration file for the --export-elasticsearch option
+# Data are available via the ES Restful API. ex: URL/<index>/cpu/system
+# https://www.elastic.co
+host=localhost
+port=9200
+index=glances
+
 [rabbitmq]
 host=localhost
 port=5672

docs/glances-doc.rst

@@ -180,10 +180,12 @@ Command-Line Options
                         set the export path for graph history
   --export-csv EXPORT_CSV
                         export stats to a CSV file
-  --export-influxdb     export stats to an InfluxDB server (influxdb needed)
-  --export-opentsdb     export stats to an OpenTSDB server (potsdb needed)
-  --export-statsd       export stats to a StatsD server (statsd needed)
-  --export-rabbitmq     export stats to a RabbitMQ server (pika needed)
+  --export-influxdb     export stats to an InfluxDB server (influxdb lib  needed)
+  --export-opentsdb     export stats to an OpenTSDB server (potsdb lib needed)
+  --export-statsd       export stats to a StatsD server (statsd lib needed)
+  --export-elasticsearch
+                        export stats to an Elasticsearch server (elasticsearch lib needed)
+  --export-rabbitmq     export stats to a RabbitMQ server (pika lib needed)
   -c CLIENT, --client CLIENT
                         connect to a Glances server by IPv4/IPv6 address or
                         hostname
@@ -953,6 +955,33 @@ Glances will generate stats as:
     'glances.load.min1': 0.19,
     ...
 
+ElasticSearch
+-------------
+
+You can export statistics to an ``ElasticSearch`` server.
+The connection should be defined in the Glances configuration file as
+following:
+
+.. code-block:: ini
+
+    [elasticsearch]
+    host=localhost
+    port=9200
+    index=glances
+
+and run Glances with:
+
+.. code-block:: console
+
+    $ glances --export-elasticsearch
+
+The stats are available through the ElasticSearch API: URL/glances/cpu/system (example for the system CPU).
+
+.. code-block:: console
+
+    $ curl  http://172.17.0.2:9200/glances/cpu/system
+    {"_index":"glances","_type":"cpu","_id":"system","_version":28,"found":true,"_source":{"timestamp": "2016-02-04T14:11:02.362232", "value": "2.2"}}
+
 RabbitMQ
 --------
 

glances/exports/glances_elasticsearch.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of Glances.
+#
+# Copyright (C) 2015 Nicolargo <nicolas@nicolargo.com>
+#
+# Glances is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Glances is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""ElasticSearch interface class."""
+
+import sys
+from datetime import datetime
+
+from glances.compat import NoOptionError, NoSectionError
+from glances.logger import logger
+from glances.exports.glances_export import GlancesExport
+
+from elasticsearch import Elasticsearch, helpers
+
+
+class Export(GlancesExport):
+
+    """This class manages the ElasticSearch (ES) export module."""
+
+    def __init__(self, config=None, args=None):
+        """Init the ES export IF."""
+        super(Export, self).__init__(config=config, args=args)
+
+        # Load the ES configuration file
+        self.host = None
+        self.port = None
+        self.index = None
+        self.export_enable = self.load_conf()
+        if not self.export_enable:
+            sys.exit(2)
+
+        # Init the ES client
+        self.client = self.init()
+
+    def load_conf(self, section="elasticsearch"):
+        """Load the ES configuration in the Glances configuration file."""
+        if self.config is None:
+            return False
+        try:
+            self.host = self.config.get_value(section, 'host')
+            self.port = self.config.get_value(section, 'port')
+            self.index = self.config.get_value(section, 'index')
+        except NoSectionError:
+            logger.critical("No ElasticSearch configuration found")
+            return False
+        except NoOptionError as e:
+            logger.critical("Error in the ElasticSearch configuration (%s)" % e)
+            return False
+        else:
+            logger.debug("Load ElasticSearch from the Glances configuration file")
+
+        return True
+
+    def init(self):
+        """Init the connection to the ES server."""
+        if not self.export_enable:
+            return None
+
+        try:
+            es = Elasticsearch(hosts=['{0}:{1}'.format(self.host, self.port)])
+        except Exception as e:
+            logger.critical("Cannot connect to ElasticSearch server %s:%s (%s)" % (self.host, self.port, e))
+            sys.exit(2)
+        else:
+            logger.info("Connected to the ElasticSearch server %s:%s" % (self.host, self.port))
+
+        try:
+            index_count = es.count(index=self.index)['count']
+        except Exception as e:
+            # Index did not exist, it will be created at the first write
+            # Create it...
+            es.indices.create(self.index)
+        else:
+            logger.info("There is already %s entries in the ElasticSearch %s index" % (index_count, self.index))
+
+        return es
+
+    def export(self, name, columns, points):
+        """Write the points to the ES server."""
+        logger.debug("Export {0} stats to ElasticSearch".format(name))
+
+        # Create DB input
+        # http://elasticsearch-py.readthedocs.org/en/master/helpers.html
+        actions = []
+        for c, p in zip(columns, points):
+            action = {
+                "_index": self.index,
+                "_type": name,
+                "_id": c,
+                "_source": {
+                    "value": str(p),
+                    "timestamp": datetime.now()
+                    }
+            }
+            actions.append(action)
+
+        # Write input to the ES index
+        try:
+            helpers.bulk(self.client, actions)
+        except Exception as e:
+            logger.error("Cannot export {0} stats to ElasticSearch ({1})".format(name, e))

glances/logger.py

@@ -70,7 +70,15 @@ LOGGING_CFG = {
         'standard': {
             'handlers': ['file'],
             'level': 'INFO'
-        }
+        },
+        'elasticsearch': {
+            'handlers': ['file', 'console'],
+            'level': 'ERROR',
+        },
+        'elasticsearch.trace': {
+            'handlers': ['file', 'console'],
+            'level': 'ERROR',
+        },
     }
 }
 

glances/main.py

@@ -152,13 +152,15 @@ Start the client browser (browser mode):\n\
         parser.add_argument('--export-csv', default=None,
                             dest='export_csv', help='export stats to a CSV file')
         parser.add_argument('--export-influxdb', action='store_true', default=False,
-                            dest='export_influxdb', help='export stats to an InfluxDB server (influxdb needed)')
+                            dest='export_influxdb', help='export stats to an InfluxDB server (influxdb lib needed)')
         parser.add_argument('--export-opentsdb', action='store_true', default=False,
-                            dest='export_opentsdb', help='export stats to an OpenTSDB server (potsdb needed)')
+                            dest='export_opentsdb', help='export stats to an OpenTSDB server (potsdb lib needed)')
         parser.add_argument('--export-statsd', action='store_true', default=False,
-                            dest='export_statsd', help='export stats to a StatsD server (statsd needed)')
+                            dest='export_statsd', help='export stats to a StatsD server (statsd lib needed)')
+        parser.add_argument('--export-elasticsearch', action='store_true', default=False,
+                            dest='export_elasticsearch', help='export stats to an ElasticSearch server (elasticsearch lib needed)')
         parser.add_argument('--export-rabbitmq', action='store_true', default=False,
-                            dest='export_rabbitmq', help='export stats to rabbitmq broker (pika needed)')
+                            dest='export_rabbitmq', help='export stats to rabbitmq broker (pika lib needed)')
         # Client/Server option
         parser.add_argument('-c', '--client', dest='client',
                             help='connect to a Glances server by IPv4/IPv6 address or hostname')
@@ -299,7 +301,7 @@ Start the client browser (browser mode):\n\
         self.args = args
 
         # Export is only available in standalone or client mode (issue #614)
-        export_tag = args.export_csv or args.export_statsd or args.export_influxdb or args.export_opentsdb or args.export_rabbitmq
+        export_tag = args.export_csv or args.export_elasticsearch or args.export_statsd or args.export_influxdb or args.export_opentsdb or args.export_rabbitmq
         if not (self.is_standalone() or self.is_client()) and export_tag:
             logger.critical("Export is only available in standalone or client mode")
             sys.exit(2)

man/glances.1

@@ -109,13 +109,16 @@ set the export path for graph history
 export stats to a CSV file
 .TP
 .B \-\-export-influxdb
-export stats to an InfluxDB server (influxdb needed)
+export stats to an InfluxDB server (influxdb lib needed)
 .TP
 .B \-\-export-opentsdb
-export stats to an OpenTSDB server (potsdb needed)
+export stats to an OpenTSDB server (potsdb lib needed)
 .TP
 .B \-\-export-statsd
-export stats to a StatsD server (statsd needed)
+export stats to a StatsD server (statsd lib needed)
+.TP
+.B \-\-export-elasticsearch
+export stats to an ElasticSearch server (elasticsearch lib needed)
 .TP
 .B \-\-export-rabbitmq
 export stats to a RabbitMQ server (pika needed)