Merge pull request #39 from Ensembl/features/central-es-connexion

Features/central es connexion

.travis.yml

@@ -9,24 +9,3 @@ install:
 script:
 - coverage run -m pytest
 - coverage report -m
-
-deploy:
-  - provider: pypi
-    username: __token__
-    server: https://test.pypi.org/legacy/
-    password:
-      secure: UCGgJqX0irr4E/Nw+hOWZojH4oj17a92opGGGMUCFsIN7vaOW7+rdEqLmvjR3cF3aTD2taMpDMBOIHOKy9xzv448FvA89ICpqbASaFhkb63UrCNCT8CMgBjrQyCqgONnlNlCDx8Od3LEd3muKUUJUzqWFVBTKD/OwFut7+Ju9rUnf5dYLUqQ1Mxl+T93G3wZtp7xCw1AyevEuF3adbDiukJDz93BGG3YiUlpIIg2DLJZMat0++O+4phCQKro6iYqbWWwCpj2t9BKeCqD7X4jG+ND3++Kg8K9SKTzpIWc5Rnk/ohuzTE0cDncrtR9HBopPaZwVyUzQkXJdIrjiW3wxPrh15SGWhpgmjvpRPkTYOfOnCB6JnvBGsErZ/sC/V5yT7u8btZ4Vi2Q8fTfThAU/ULD/15DtmHl08Hc+R+dV0gkgCCEVNUHBP65rlLKjXwHjItDlzpVtUBkDMH14E7QgN0X+lrysMXFs01MxcxLhGw8yL08XUPUjdoBgLWLTpVkR2rrhDvXSaIVa/OLJXILRfvdHkh5VoiffVqVyAmscUJnN8YTFi304aQix6/Ubs1Ajiq2oubF7tML9XCErL24IHzOkXelmtipmzO8uUV2WSxakLRX+EH6L7Cccvgba8cYFcsurgJxF29CU1ZAAHWhHGfpf2Y4pmZyRmDl0SP6YDw=
-    on:
-      tags: true
-      repo: Ensembl/ensembl-prodinf-core
-      condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?-test(\.[0-9]+)?$
-    skip_existing: true
-  - provider: pypi
-    username: __token__
-    password:
-      secure: oStGO0suyyGYgYU2cckOz/0o2RgdTg2Kxg3Xs8c6M/V+BbffgKk9RNYCtSPmggXdMO4esKdhn8nHWzG8P49XiWo11v/W7O6N6DrJSLpk/Pt4P8u2U92TK6YLHKzvGd3LNVI42QOk4XaC63w4vkXU0afd5g2Fhflm3Hk4uEzYfRRhViJBm/SYt+/pQuZTpyGO7sh0s0VLVc1l3TMMyAvvuU6yQro1B7DugiwJqLVuV28qCZGIykbk5j6nXsNngmbVfb+Pxgu3j8XQAMh8KTWlZIW92iNWJyFVx6wRleM/NRcrWHzylq8flRYV3pbY/uZaGsd6x07NekvqVOz33gZJi7DiYSPJBf4S1ZIGorprlTj0+Y3IAOW9Y1MMmSxurjAQGZ69F8SLiuzUpd2GJaDFRP3nA2fzPzSwcFmVKzTSl2Ey8bpz8VDF33Zs6PM4sBg8EcE2PTRueJ14Kf5lDm5RbhQMFDqU4iMPPDlDBDNc3C1AsgQbrDE4I9YG893MruNgjIADIGMuqOcF/j0Dq5if/IfBCh2Au/JWLpejZgS6upr3m9NoyWWbmOYG3lc2k1gDaKS/B45NypbOvz43EwSbs5ajEkvekZpZJ3E/OUtcURNUctqZM7rEkwU2VtAxyHLLkUY+/65ZZK5GK+j5xbFV5RtayH/OX4zODvlBz6l3PC4=
-    on:
-      tags: true
-      repo: Ensembl/ensembl-prodinf-core
-      condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$
-    skip_existing: true

requirements-test.txt

@@ -1,2 +1,3 @@
 pytest
+elasticsearch>=6.8.1,<7.0.0
 coverage[toml]

setup.py

@@ -9,6 +9,7 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
+import os
 import pathlib
 from setuptools import setup, find_namespace_packages
 
@@ -27,7 +28,7 @@ def import_requirements():
     name='ensembl-prodinf-core',
     description='Ensembl Production infrastructure core package',
     long_description=readme,
-    version=version,
+    version=os.getenv('CI_COMMIT_TAG', version),
     namespace_packages=['ensembl'],
     packages=find_namespace_packages(where='src', include=['ensembl*'], exclude=['test']),
     package_dir={'': 'src'},

src/ensembl/production/core/es.py

@@ -0,0 +1,43 @@
+# See the NOTICE file distributed with this work for additional information
+#    regarding copyright ownership.
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+import ssl
+
+import urllib3
+from elasticsearch import Elasticsearch, ElasticsearchException
+from elasticsearch.connection import create_ssl_context
+
+
+class ElasticsearchConnectionManager:
+    def __init__(self, host: str, port: int, user: str = "", password: str = "", with_ssl: bool = False):
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.ssl = with_ssl
+        self.client = None
+
+    def __enter__(self):
+        urllib3.disable_warnings(category=urllib3.connectionpool.InsecureRequestWarning)
+        ssl_context = create_ssl_context()
+        ssl_context.check_hostname = False
+        ssl_context.verify_mode = ssl.CERT_NONE
+        self.client = Elasticsearch(hosts=[{'host': self.host, 'port': self.port}],
+                                    scheme="https" if self.ssl else "http",
+                                    ssl_context=ssl_context,
+                                    http_auth=(self.user, self.password))
+        if not self.client.ping():
+            raise RuntimeError(
+                f"Cannot connect to Elasticsearch server. User: {self.user}, Host: {self.host}:{self.port}")
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        self.client.transport.close()

src/test/test_es.py

@@ -0,0 +1,25 @@
+#    See the NOTICE file distributed with this work for additional information
+#    regarding copyright ownership.
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+import unittest
+
+from ensembl.production.core.es import ElasticsearchConnectionManager
+
+
+# TODO ADD more test with proper config in gitlab
+class EsTest(unittest.TestCase):
+    def test_es_exception(self):
+        with self.assertRaises(RuntimeError) as expected:
+            with ElasticsearchConnectionManager(host="http://unknown",
+                                                port=9200) as elastic:
+                elastic.client.ping()
+            self.assertTrue('Cannot connect to Elasticsearch server' in expected)
+