From 054c0e25129bb271eed12659ff9d2286ef5e6381 Mon Sep 17 00:00:00 2001 From: Manubens Date: Wed, 26 Oct 2022 09:44:24 +0000 Subject: [PATCH 01/10] quick fix for POLY-244 and version bump --- polytope_server/version.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/polytope_server/version.py b/polytope_server/version.py index d0700a1..e7e981e 100644 --- a/polytope_server/version.py +++ b/polytope_server/version.py @@ -20,4 +20,4 @@ # Single-source Polytope version number -__version__ = "0.7.6" +__version__ = "0.7.7" diff --git a/requirements.txt b/requirements.txt index ef4bbc1..5cd149d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +markupsafe<2.1.0 flask>=1.0.0 flask-script==2.0.5 flask-mongoengine==0.8.2 From c4aaad32eaaf214ac1f57eeea2ccdbf7c099e406 Mon Sep 17 00:00:00 2001 From: majh Date: Thu, 19 Jan 2023 00:23:24 +0000 Subject: [PATCH 02/10] Update dependency versions --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5cd149d..e461a9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -markupsafe<2.1.0 -flask>=1.0.0 +Jinja2==3.1.1 +Flask==2.0.3 flask-script==2.0.5 flask-mongoengine==0.8.2 py-bcrypt==0.4 @@ -7,8 +7,8 @@ jsonschema==2.5.1 pika==1.1.0 requests==2.22.0 attrdict==2.0.1 -pytest==5.0.1 -pyyaml>=5.4 +pytest>=5.0.1 +pyyaml==5.1.1 hiyapyco==0.4.14 pykwalify==1.7.0 falcon==2.0.0 @@ -16,7 +16,7 @@ falcon-jsonify==1.2 flask-restful==0.3.7 flask-restplus==0.12.1 flask-wtf==0.14.2 -werkzeug==0.15.5 +werkzeug==2.0 gunicorn==19.9.0 ecmwf-api-client==1.5.4 pymongo==3.10.1 @@ -29,4 +29,4 @@ deepmerge==0.1.0 flask-swagger-ui==3.25.0 ldap3==2.7 docker==4.2.0 -python-keycloak==0.24.0 +python-keycloak==0.24.0 \ No newline at end of file From 17518d47b4bfe2630ee33d18d73fe7b65f4bcea0 Mon Sep 17 00:00:00 2001 From: jameshawkes Date: Thu, 30 Mar 2023 19:52:18 +0100 Subject: [PATCH 03/10] more flexible ldap --- .../authorization/ldap_authorization.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/polytope_server/common/authorization/ldap_authorization.py b/polytope_server/common/authorization/ldap_authorization.py index c450a93..f062c29 100644 --- a/polytope_server/common/authorization/ldap_authorization.py +++ b/polytope_server/common/authorization/ldap_authorization.py @@ -34,6 +34,8 @@ def __init__(self, name, realm, config): self.url = config.get("url") self.search_base = config.get("search_base") self.filter = config.get("filter", "") + self.ldap_user = config.get("ldap_user", "") + self.ldap_password = config.get("ldap_password", "") # Alternative attribute to use instead of user's username self.username_attribute = config.get("username-attribute", None) @@ -46,7 +48,7 @@ def get_roles(self, user: User) -> list: ) try: if self.username_attribute is None: - return retrieve_ldap_user_roles(user.username, self.filter, self.url, self.search_base) + return retrieve_ldap_user_roles(user.username, self.filter, self.url, self.search_base, self.ldap_user, self.ldap_password) else: return retrieve_ldap_user_roles( user.attributes[self.username_attribute], @@ -67,28 +69,32 @@ def collect_metric_info(self): ################################################# -def retrieve_ldap_user_roles(uid, filter, url, search_base): +def retrieve_ldap_user_roles(uid: str, filter: str, url: str, search_base: str, ldap_user: str, ldap_password: str) -> list: """ - Takes an ECMWF UID and returns all roles matching - the provided filter 'filter'. + Takes an ECMWF UID and returns all roles matching + the provided filter 'filter'. """ server = Server(url) - connection = Connection(server) + connection = Connection( + server, + user="CN={},OU=Connectors,OU=Service Accounts,DC=ecmwf,DC=int".format(ldap_user), + password=ldap_password, + raise_exceptions=True + ) with connection as conn: - logging.debug("Looking up {} in LDAP".format(uid)) conn.search( search_base=search_base, - search_filter="(&(objectClass=person)(cn={}))".format(uid), + search_filter='(&(objectClass=person)(cn={}))'.format(uid), search_scope=SUBTREE, - attributes=["memberOf"], + attributes=['memberOf'] ) user_data = json.loads(conn.response_to_json()) - if len(user_data["entries"]) == 0: - raise KeyError("User {} not found in LDAP.".format(uid)) - roles = user_data["entries"][0]["attributes"]["memberOf"] + if len(user_data['entries']) == 0: + raise KeyError('User {} not found in LDAP.'.format(uid)) + roles = user_data['entries'][0]['attributes']['memberOf'] # Filter roles matches = [] @@ -98,9 +104,7 @@ def retrieve_ldap_user_roles(uid, filter, url, search_base): # Parse CN=x,OU=y,OU=z,... into dict and extract 'common name' (CN) for i, role in enumerate(matches): - d = dict(s.split("=") for s in role.split(",")) - matches[i] = d["CN"] - - logging.debug("User {} has roles: {}".format(uid, matches)) + d = dict(s.split('=') for s in role.split(',')) + matches[i] = d['CN'] return matches From 088c7bb4d6df8a3669701821455d7301a3b9a657 Mon Sep 17 00:00:00 2001 From: jameshawkes Date: Thu, 18 May 2023 09:13:29 +0100 Subject: [PATCH 04/10] data path default update --- tests/unit/test_mars_match.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_mars_match.py b/tests/unit/test_mars_match.py index 65404ff..1d58517 100644 --- a/tests/unit/test_mars_match.py +++ b/tests/unit/test_mars_match.py @@ -34,7 +34,7 @@ def setup_method(self, method): polytope_config.global_config["datasources"]["mars"] = { "type": "mars", "command": "mars", - "tmp_dir": "/data", + "tmp_dir": "/home/polytope/data", "match": None, } From 9d4ffa4fd2a5b8f06353fde515e4466b6df2ea86 Mon Sep 17 00:00:00 2001 From: jameshawkes Date: Wed, 14 Jun 2023 08:38:19 +0100 Subject: [PATCH 05/10] mars-cpp --- polytope_server/common/datasource/mars.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/polytope_server/common/datasource/mars.py b/polytope_server/common/datasource/mars.py index 7175116..c24e20c 100644 --- a/polytope_server/common/datasource/mars.py +++ b/polytope_server/common/datasource/mars.py @@ -48,6 +48,15 @@ def __init__(self, config): if self.match_rules is None: self.match_rules = {} + self.mars_binary = config.get("binary", "mars") + + # Write the mars config + self.mars_config = config.get("config", {}) + self.mars_home = self.tmp_dir + "/mars-home" + os.makedirs(self.mars_home + "/etc/mars-client/") + with open(self.mars_home + "/etc/mars-client/databases.yaml", "w") as f: + yaml.dump(self.mars_config, f) + def get_type(self): return self.type @@ -155,6 +164,8 @@ def make_env(self, request): **os.environ, "MARS_USER_EMAIL": mars_user, "MARS_USER_TOKEN": mars_token, + "MARS_HOME": self.mars_home, + "ECMWF_MARS_COMMAND": self.mars_binary } logging.info("Accessing MARS on behalf of user {} with token {}".format(mars_user, mars_token)) From b87e386b9ee1c03837e13e16492ddfa69d2adc96 Mon Sep 17 00:00:00 2001 From: jameshawkes Date: Wed, 14 Jun 2023 08:44:36 +0100 Subject: [PATCH 06/10] 200MiB upload chunks --- polytope_server/common/staging/s3_staging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_server/common/staging/s3_staging.py b/polytope_server/common/staging/s3_staging.py index f53fb45..bac873c 100644 --- a/polytope_server/common/staging/s3_staging.py +++ b/polytope_server/common/staging/s3_staging.py @@ -82,7 +82,7 @@ def create(self, name, data, content_type): i = 1 parts = {} total_size = 0 - for buf in self.iterator_buffer(data, 6 * 1024 * 1024): + for buf in self.iterator_buffer(data, 200 * 1024 * 1024): if len(buf) == 0: break try: From 3fe8368a57da7fe70d5e6d2d4b9bec18aa609cda Mon Sep 17 00:00:00 2001 From: jameshawkes Date: Wed, 14 Jun 2023 08:48:10 +0100 Subject: [PATCH 07/10] 200MiB upload chunk size --- polytope_server/common/staging/s3_staging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_server/common/staging/s3_staging.py b/polytope_server/common/staging/s3_staging.py index f53fb45..bac873c 100644 --- a/polytope_server/common/staging/s3_staging.py +++ b/polytope_server/common/staging/s3_staging.py @@ -82,7 +82,7 @@ def create(self, name, data, content_type): i = 1 parts = {} total_size = 0 - for buf in self.iterator_buffer(data, 6 * 1024 * 1024): + for buf in self.iterator_buffer(data, 200 * 1024 * 1024): if len(buf) == 0: break try: From 6f83c0bd1f53bebd06cb9f2a580ce0c76574d169 Mon Sep 17 00:00:00 2001 From: majh Date: Wed, 21 Jun 2023 10:53:50 +0100 Subject: [PATCH 08/10] fix mars temporary directory error when already exists --- polytope_server/common/datasource/mars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_server/common/datasource/mars.py b/polytope_server/common/datasource/mars.py index c24e20c..6a05968 100644 --- a/polytope_server/common/datasource/mars.py +++ b/polytope_server/common/datasource/mars.py @@ -53,7 +53,7 @@ def __init__(self, config): # Write the mars config self.mars_config = config.get("config", {}) self.mars_home = self.tmp_dir + "/mars-home" - os.makedirs(self.mars_home + "/etc/mars-client/") + os.makedirs(self.mars_home + "/etc/mars-client/", exist_ok=True) with open(self.mars_home + "/etc/mars-client/databases.yaml", "w") as f: yaml.dump(self.mars_config, f) From 13eeafb6937451031da314fc59a6245a423aa27c Mon Sep 17 00:00:00 2001 From: majh Date: Fri, 30 Jun 2023 12:47:17 +0100 Subject: [PATCH 09/10] make mars datasource capable of handling both mars and mars-cpp client --- polytope_server/common/datasource/mars.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/polytope_server/common/datasource/mars.py b/polytope_server/common/datasource/mars.py index 6a05968..97600c6 100644 --- a/polytope_server/common/datasource/mars.py +++ b/polytope_server/common/datasource/mars.py @@ -51,11 +51,14 @@ def __init__(self, config): self.mars_binary = config.get("binary", "mars") # Write the mars config - self.mars_config = config.get("config", {}) - self.mars_home = self.tmp_dir + "/mars-home" - os.makedirs(self.mars_home + "/etc/mars-client/", exist_ok=True) - with open(self.mars_home + "/etc/mars-client/databases.yaml", "w") as f: - yaml.dump(self.mars_config, f) + if "config" in config: + self.mars_config = config.get("config", {}) + self.mars_home = self.tmp_dir + "/mars-home" + os.makedirs(self.mars_home + "/etc/mars-client/", exist_ok=True) + with open(self.mars_home + "/etc/mars-client/databases.yaml", "w") as f: + yaml.dump(self.mars_config, f) + else: + self.mars_home = None def get_type(self): return self.type @@ -164,10 +167,15 @@ def make_env(self, request): **os.environ, "MARS_USER_EMAIL": mars_user, "MARS_USER_TOKEN": mars_token, - "MARS_HOME": self.mars_home, "ECMWF_MARS_COMMAND": self.mars_binary } + if self.mars_config is not None: + env = { + **os.environ, + "MARS_HOME": self.mars_home, + } + logging.info("Accessing MARS on behalf of user {} with token {}".format(mars_user, mars_token)) except Exception: From 735a04388a17b3bd6c460b7ba0c84a2c4d6b5ebf Mon Sep 17 00:00:00 2001 From: majh Date: Fri, 30 Jun 2023 12:50:23 +0100 Subject: [PATCH 10/10] bump version --- polytope_server/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_server/version.py b/polytope_server/version.py index e7e981e..2290349 100644 --- a/polytope_server/version.py +++ b/polytope_server/version.py @@ -20,4 +20,4 @@ # Single-source Polytope version number -__version__ = "0.7.7" +__version__ = "0.7.8"