Merge pull request #3 from opsdis/async

Merging async branch with master - enable async based handling of the exporter requests

README.md

@@ -7,6 +7,9 @@ icinga2-exporter
 The icinga2-exporter utilizes the [Icinga 2](https://icinga.com/) REST API to fetch service based performance
 data and publish it in a way that lets [Prometheus](https://prometheus.io/) scrape the performance data as metrics.
 
+The service is based on [Quart](https://pgjones.gitlab.io/quart/). Quart's is compatible with Flask but based 
+on Asyncio.  
+
 Benefits:
 
 - Enable advanced queries and aggregation on timeseries
@@ -81,7 +84,7 @@ All configuration is made in the config.yml file.
 Example:
 ```yaml
 
-# Port can be overridden by using -p if running development flask
+# Port can be overridden by using -p if running development quart
 #port: 9638
 
 icinga2:
@@ -189,21 +192,21 @@ scrape_configs:
 
 # Running
 
-## Development with flask built in webserver
+## Development with quart built in webserver
 
     python -m  icinga2_exporter -f config.yml
 
 The switch -p enable setting of the port.
 
-## Production with gunicorn
+## Production with gunicorn as ASGI continer 
 
 Running with default config.yml. The default location is current directory
 
-    gunicorn --access-logfile /dev/null -w 4 "wsgi:create_app()"
+    gunicorn --access-logfile /dev/null -w 4 -k uvicorn.workers.UvicornWorker "wsgi:create_app()"
 
 Set the path to the configuration file.
 
-    gunicorn --access-logfile /dev/null -w 4 "wsgi:create_app('/etc/icinga2-exporter/config.yml')"
+    gunicorn --access-logfile /dev/null -w 4 -k uvicorn.workers.UvicornWorker "wsgi:create_app('/etc/icinga2-exporter/config.yml')"
 
 > Port for gunicorn is default 8000, but can be set with -b, e.g. `-b localhost:9638`
 

icinga2_exporter/log.py

@@ -23,7 +23,7 @@
 import datetime
 from pythonjsonlogger import jsonlogger
 
-logger = logging.getLogger('monitor-exporter')
+logger = logging.getLogger('icinga2-exporter')
 
 
 # def configure_logger(log_level="INFO", log_filename=None, format=None):

icinga2_exporter/main.py

@@ -20,11 +20,10 @@
 """
 
 import argparse
+from quart import Quart
 import icinga2_exporter.log as log
 import icinga2_exporter.fileconfiguration as config
-import icinga2_exporter.proxy as proxy
-
-from flask import Flask
+from icinga2_exporter.proxy import app as icinga2
 import icinga2_exporter.monitorconnection as monitorconnection
 
 
@@ -57,14 +56,13 @@ def start():
         port = args.port
 
     log.configure_logger(configuration)
-    ##
 
     monitorconnection.MonitorConfig(configuration)
     log.info('Starting web app on port: ' + str(port))
 
-    app = Flask(__name__)
-
-    app.register_blueprint(proxy.app, url_prefix='/')
+    app = Quart(__name__)
+    app.register_blueprint(icinga2, url_prefix='')
+    list_routes(app)
     app.run(host='0.0.0.0', port=port)
 
 
@@ -86,8 +84,14 @@ def create_app(config_path=None):
     monitorconnection.MonitorConfig(configuration)
     log.info('Starting web app')
 
-    app = Flask(__name__)
-
-    app.register_blueprint(proxy.app, url_prefix='/')
+    app = Quart(__name__)
+    app.register_blueprint(icinga2, url_prefix='')
 
     return app
+
+def list_routes(app):
+    import urllib
+    output = []
+    for rule in app.url_map.iter_rules():
+        print("route")
+        print (rule)

icinga2_exporter/monitorconnection.py

@@ -21,6 +21,7 @@
 
 import requests
 import json
+import aiohttp
 from requests.auth import HTTPBasicAuth
 import icinga2_exporter.log as log
 
@@ -142,3 +143,32 @@ def post(self, url, body):
             log.error("{}".format(str(err)))
 
         return data_json
+
+
+    async def async_get_perfdata(self, hostname):
+        # Get performance data from Monitor and return in json format
+        body = {"joins": ["host.vars"],
+                "attrs": ["__name", "display_name", "check_command", "last_check_result", "vars", "host_name"],
+                "filter": 'service.host_name==\"{}\"'.format(hostname)}
+
+        data_json = await self.async_post(self.url_query_service_perfdata, body)
+
+        if not data_json:
+            log.warn('Received no perfdata from Icinga2')
+
+        return data_json
+
+
+    async def async_post(self, url, body):
+        data_json = {}
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(url, auth=aiohttp.BasicAuth(self.user, self.passwd),
+                                        verify_ssl=False,
+                                        headers={'Content-Type': 'application/json',
+                                                 'X-HTTP-Method-Override': 'GET'},
+                                        data=json.dumps(body)) as response:
+                    re = await response.text()
+                    return json.loads(re)
+        finally:
+            pass

icinga2_exporter/perfdata.py

@@ -42,12 +42,13 @@ def __init__(self, monitor: Monitor, query_hostname: str):
         self.perfname_to_label = monitor.get_perfname_to_label()
         self.perfdatadict = {}
 
-    def get_perfdata(self) -> dict:
+    async def get_perfdata(self) -> dict:
         """
         Collect icinga2 data and parse it into prometheus metrics
         :return:
         """
-        data_json = self.monitor.get_perfdata(self.query_hostname)
+
+        data_json = await self.monitor.async_get_perfdata(self.query_hostname)
         if 'results' in data_json:
             for serivce_attrs in data_json['results']:
                 if 'attrs' in serivce_attrs and 'last_check_result' in serivce_attrs['attrs'] and 'performance_data' in \
@@ -61,6 +62,7 @@ def get_perfdata(self) -> dict:
                     # For all host custom vars add as label
                     labels.update(Perfdata.get_host_custom_vars(serivce_attrs))
 
+
                     for perf_string in serivce_attrs['attrs']['last_check_result']['performance_data']:
                         perf = Perfdata.parse_perfdata(perf_string)
 

icinga2_exporter/proxy.py

@@ -18,25 +18,26 @@
     along with icinga2-exporter-exporter.  If not, see <http://www.gnu.org/licenses/>.
 
 """
+import asyncio
+from quart import request, Response, jsonify, Quart, Blueprint
 
-from flask import request, Response, jsonify, Blueprint
 from prometheus_client import (CONTENT_TYPE_LATEST, Counter)
 
 from icinga2_exporter.perfdata import Perfdata
 import icinga2_exporter.monitorconnection as monitorconnection
 import icinga2_exporter.log as log
 
-app = Blueprint("prom", __name__)
+#app = Quart( __name__)
+app = Blueprint( 'icinga2',__name__)
 total_requests = Counter('requests', 'Total requests to monitor-exporter endpoint')
 
 
 @app.route('/', methods=['GET'])
 def hello_world():
     return 'monitor-exporter alive'
 
-
 @app.route("/metrics", methods=['GET'])
-def get_metrics():
+async def get_ametrics():
     log.info(request.url)
     target = request.args.get('target')
 
@@ -45,22 +46,21 @@ def get_metrics():
     monitor_data = Perfdata(monitorconnection.MonitorConfig(), target)
 
     # Fetch performance data from Monitor
-    monitor_data.get_perfdata()
+    await asyncio.get_event_loop().create_task(monitor_data.get_perfdata())
 
     target_metrics = monitor_data.prometheus_format()
 
     resp = Response(target_metrics)
     resp.headers['Content-Type'] = CONTENT_TYPE_LATEST
-
+    #after_request_func(resp)
     return resp
 
-
 @app.route("/health", methods=['GET'])
 def get_health():
     return chech_healthy()
 
 
-@app.after_request
+#@app.after_request
 def after_request_func(response):
     total_requests.inc()
 
@@ -75,72 +75,3 @@ def chech_healthy() -> Response:
     resp = jsonify({'status': 'ok'})
     resp.status_code = 200
     return resp
-
-# def read_config(config_file: str) -> dict:
-#     """
-#     Read configuration file
-#     :param config_file:
-#     :return:
-#     """
-#     config = {}
-#     try:
-#         ymlfile = open(config_file, 'r')
-#         config = yaml.load(ymlfile, Loader=yaml.SafeLoader)
-#     except (FileNotFoundError, IOError):
-#         print("Config file {} not found".format(config_file))
-#         exit(1)
-#     except (yaml.YAMLError, yaml.MarkedYAMLError) as err:
-#         print("Error will reading config file - {}".format(err))
-#         exit(1)
-#
-#     return config
-
-
-# def start():
-#     parser = argparse.ArgumentParser(description='monitor_exporter')
-#
-#     parser.add_argument('-f', '--configfile',
-#                         dest="configfile", help="configuration file")
-#
-#     parser.add_argument('-p', '--port',
-#                         dest="port", help="Server port")
-#
-#     args = parser.parse_args()
-#
-#     port = 5000
-#     if args.port:
-#         port = args.port
-#
-#     config_file = 'config.yml'
-#     if args.configfile:
-#         config_file = args.configfile
-#
-#     configuration = config.read_config(config_file)
-#
-#     formatter = log.configure_logger(configuration)
-#     ##
-#
-#     monitorconnection.MonitorConfig(configuration)
-#     log.info('Starting web app on port: ' + str(port))
-#
-#
-#     app.run(host='0.0.0.0', port=port)
-#     app.logger.handlers[0].setFormatter(formatter)
-
-
-# def create_app(config_path = None):
-#
-#     config_file = 'config.yml'
-#     if config_path:
-#         config_file = config_path
-#
-#     config = read_config(config_file)
-#
-#     formatter = log.configure_logger(config)
-#
-#     monitorconnection.MonitorConfig(config)
-#     log.info('Starting web app')
-#
-#     app.logger.handlers[0].setFormatter(formatter)
-#
-#     return app

requirements.txt

@@ -1,17 +1,38 @@
+aiofiles==0.4.0
+aiohttp==3.5.4
+async-timeout==3.0.1
+asyncio==3.4.3
+attrs==19.1.0
+blinker==1.4
 certifi==2019.6.16
 chardet==3.0.4
 Click==7.0
 coverage==4.5.4
-Flask==1.1.1
-Flask-Ext==0.1
+dataclasses==0.6
 gunicorn==19.9.0
+h11==0.8.1
+h2==3.1.1
+hpack==3.0.0
+httptools==0.0.13
+Hypercorn==0.5.4
+hyperframe==5.2.0
 idna==2.8
+idna-ssl==1.1.0
 itsdangerous==1.1.0
 Jinja2==2.10.1
 MarkupSafe==1.1.1
+multidict==4.5.2
 prometheus-client==0.7.1
 python-json-logger==0.1.11
+pytoml==0.1.21
 PyYAML==5.1.2
+Quart==0.6.14
 requests==2.22.0
+sortedcontainers==2.1.0
+typing-extensions==3.7.4
 urllib3==1.25.3
-Werkzeug==0.15.5
+uvicorn==0.8.6
+uvloop==0.12.2
+websockets==7.0
+wsproto==0.15.0
+yarl==1.3.0