Merge pull request #3 from ecmwf-projects/develop

Release

polytope_server/common/authorization/ldap_authorization.py

@@ -34,6 +34,8 @@ def __init__(self, name, realm, config):
         self.url = config.get("url")
         self.search_base = config.get("search_base")
         self.filter = config.get("filter", "")
+        self.ldap_user = config.get("ldap_user", "")
+        self.ldap_password = config.get("ldap_password", "")
 
         # Alternative attribute to use instead of user's username
         self.username_attribute = config.get("username-attribute", None)
@@ -46,7 +48,7 @@ def get_roles(self, user: User) -> list:
             )
         try:
             if self.username_attribute is None:
-                return retrieve_ldap_user_roles(user.username, self.filter, self.url, self.search_base)
+                return retrieve_ldap_user_roles(user.username, self.filter, self.url, self.search_base, self.ldap_user, self.ldap_password)
             else:
                 return retrieve_ldap_user_roles(
                     user.attributes[self.username_attribute],
@@ -67,28 +69,32 @@ def collect_metric_info(self):
 #################################################
 
 
-def retrieve_ldap_user_roles(uid, filter, url, search_base):
+def retrieve_ldap_user_roles(uid: str, filter: str, url: str, search_base: str, ldap_user: str, ldap_password: str) -> list:
     """
-    Takes an ECMWF UID and returns all roles matching
-    the provided filter 'filter'.
+        Takes an ECMWF UID and returns all roles matching
+        the provided filter 'filter'.
     """
 
     server = Server(url)
 
-    connection = Connection(server)
+    connection = Connection(
+        server,
+        user="CN={},OU=Connectors,OU=Service Accounts,DC=ecmwf,DC=int".format(ldap_user),
+        password=ldap_password,
+        raise_exceptions=True
+    )
 
     with connection as conn:
-        logging.debug("Looking up {} in LDAP".format(uid))
         conn.search(
             search_base=search_base,
-            search_filter="(&(objectClass=person)(cn={}))".format(uid),
+            search_filter='(&(objectClass=person)(cn={}))'.format(uid),
             search_scope=SUBTREE,
-            attributes=["memberOf"],
+            attributes=['memberOf']
         )
         user_data = json.loads(conn.response_to_json())
-        if len(user_data["entries"]) == 0:
-            raise KeyError("User {} not found in LDAP.".format(uid))
-        roles = user_data["entries"][0]["attributes"]["memberOf"]
+        if len(user_data['entries']) == 0:
+            raise KeyError('User {} not found in LDAP.'.format(uid))
+        roles = user_data['entries'][0]['attributes']['memberOf']
 
     # Filter roles
     matches = []
@@ -98,9 +104,7 @@ def retrieve_ldap_user_roles(uid, filter, url, search_base):
 
     # Parse CN=x,OU=y,OU=z,... into dict and extract 'common name' (CN)
     for i, role in enumerate(matches):
-        d = dict(s.split("=") for s in role.split(","))
-        matches[i] = d["CN"]
-
-    logging.debug("User {} has roles: {}".format(uid, matches))
+        d = dict(s.split('=') for s in role.split(','))
+        matches[i] = d['CN']
 
     return matches

polytope_server/common/datasource/mars.py

@@ -48,6 +48,18 @@ def __init__(self, config):
         if self.match_rules is None:
             self.match_rules = {}
 
+        self.mars_binary = config.get("binary", "mars")
+
+        # Write the mars config
+        if "config" in config:
+            self.mars_config = config.get("config", {})
+            self.mars_home = self.tmp_dir + "/mars-home"
+            os.makedirs(self.mars_home + "/etc/mars-client/", exist_ok=True)
+            with open(self.mars_home + "/etc/mars-client/databases.yaml", "w") as f:
+                yaml.dump(self.mars_config, f)
+        else:
+            self.mars_home = None
+
     def get_type(self):
         return self.type
 
@@ -155,8 +167,15 @@ def make_env(self, request):
                 **os.environ,
                 "MARS_USER_EMAIL": mars_user,
                 "MARS_USER_TOKEN": mars_token,
+                "ECMWF_MARS_COMMAND": self.mars_binary
             }
 
+            if self.mars_config is not None:
+                env = {
+                    **os.environ,
+                    "MARS_HOME": self.mars_home,
+                }
+
             logging.info("Accessing MARS on behalf of user {} with token {}".format(mars_user, mars_token))
 
         except Exception:

polytope_server/common/staging/s3_staging.py

@@ -82,7 +82,7 @@ def create(self, name, data, content_type):
         i = 1
         parts = {}
         total_size = 0
-        for buf in self.iterator_buffer(data, 6 * 1024 * 1024):
+        for buf in self.iterator_buffer(data, 200 * 1024 * 1024):
             if len(buf) == 0:
                 break
             try:

polytope_server/version.py

@@ -20,4 +20,4 @@
 
 
 # Single-source Polytope version number
-__version__ = "0.7.6"
+__version__ = "0.7.8"

requirements.txt

@@ -1,21 +1,22 @@
-flask>=1.0.0
+Jinja2==3.1.1
+Flask==2.0.3
 flask-script==2.0.5
 flask-mongoengine==0.8.2
 py-bcrypt==0.4
 jsonschema==2.5.1
 pika==1.1.0
 requests==2.22.0
 attrdict==2.0.1
-pytest==5.0.1
-pyyaml>=5.4
+pytest>=5.0.1
+pyyaml==5.1.1
 hiyapyco==0.4.14
 pykwalify==1.7.0
 falcon==2.0.0
 falcon-jsonify==1.2
 flask-restful==0.3.7
 flask-restplus==0.12.1
 flask-wtf==0.14.2
-werkzeug==0.15.5
+werkzeug==2.0
 gunicorn==19.9.0
 ecmwf-api-client==1.5.4
 pymongo==3.10.1
@@ -28,4 +29,4 @@ deepmerge==0.1.0
 flask-swagger-ui==3.25.0
 ldap3==2.7
 docker==4.2.0
-python-keycloak==0.24.0
+python-keycloak==0.24.0

tests/unit/test_mars_match.py

@@ -34,7 +34,7 @@ def setup_method(self, method):
         polytope_config.global_config["datasources"]["mars"] = {
             "type": "mars",
             "command": "mars",
-            "tmp_dir": "/data",
+            "tmp_dir": "/home/polytope/data",
             "match": None,
         }