From 76b0a250b723b5d72cc161e9432b0dd25c476ce2 Mon Sep 17 00:00:00 2001 From: Adrien Perrin Date: Fri, 22 Sep 2023 13:39:14 +0000 Subject: [PATCH 1/3] add TOTP authentication for the HTTP downloader --- geospaas_processing/downloaders.py | 42 ++++++++++++---- geospaas_processing/provider_settings.yml | 11 +++- tests/test_downloaders.py | 61 ++++++++++++++++++++++- 3 files changed, 100 insertions(+), 14 deletions(-) diff --git a/geospaas_processing/downloaders.py b/geospaas_processing/downloaders.py index e0d9fa3..13d3723 100644 --- a/geospaas_processing/downloaders.py +++ b/geospaas_processing/downloaders.py @@ -12,11 +12,13 @@ import logging import os import os.path +import re import shutil from urllib.parse import urlparse import oauthlib.oauth2 -import re +import oauthlib.oauth2.rfc6749.errors +import pyotp import requests import requests.utils import requests_oauthlib @@ -161,16 +163,33 @@ class HTTPDownloader(Downloader): CHUNK_SIZE = 1024 * 1024 @classmethod - def build_oauth2_authentication(cls, username, password, token_url, client_id): + def build_oauth2_authentication(cls, username, password, token_url, client_id, + totp_secret=None): """Creates an OAuth2 object usable by `requests` methods""" - client = oauthlib.oauth2.LegacyApplicationClient(client_id=client_id) - token = requests_oauthlib.OAuth2Session(client=client).fetch_token( - token_url=token_url, - username=username, - password=password, - client_id=client_id, - ) - return requests_oauthlib.OAuth2(client_id=client_id, client=client, token=token) + # TOTP passwords are valid for 30 seconds, so we retry a few + # times in case we get unlucky and the password expires between + # the generation of the password and the authentication request + retries = 5 + while retries > 0: + client = oauthlib.oauth2.LegacyApplicationClient(client_id=client_id) + session_args = { + 'token_url': token_url, + 'username': username, + 'password': password, + 'client_id': client_id, + } + if totp_secret: + session_args['totp'] = pyotp.TOTP(totp_secret).now() + + try: + token = requests_oauthlib.OAuth2Session(client=client).fetch_token(**session_args) + except oauthlib.oauth2.rfc6749.errors.InvalidGrantError: + retries -= 1 + if retries > 0: + continue + else: + raise + return requests_oauthlib.OAuth2(client_id=client_id, client=client, token=token) @classmethod def get_auth(cls, kwargs): @@ -185,7 +204,8 @@ def get_auth(cls, kwargs): kwargs['username'], kwargs['password'], kwargs['token_url'], - kwargs['client_id'] + kwargs['client_id'], + totp_secret=kwargs.get('totp_secret'), ) else: return super().get_auth(kwargs) diff --git a/geospaas_processing/provider_settings.yml b/geospaas_processing/provider_settings.yml index 9f82732..9b839a1 100644 --- a/geospaas_processing/provider_settings.yml +++ b/geospaas_processing/provider_settings.yml @@ -31,9 +31,18 @@ username: !ENV 'CREODIAS_USERNAME' password: !ENV 'CREODIAS_PASSWORD' authentication_type: 'oauth2' - token_url: 'https://identity.cloudferro.com/auth/realms/DIAS/protocol/openid-connect/token' + token_url: 'https://identity.cloudferro.com/auth/realms/Creodias-new/protocol/openid-connect/token' client_id: 'CLOUDFERRO_PUBLIC' max_parallel_downloads: 10 + totp_secret: !ENV 'CREODIAS_TOTP_SECRET' +'https://datahub.creodias.eu': + username: !ENV 'CREODIAS_USERNAME' + password: !ENV 'CREODIAS_PASSWORD' + authentication_type: 'oauth2' + token_url: 'https://identity.cloudferro.com/auth/realms/Creodias-new/protocol/openid-connect/token' + client_id: 'CLOUDFERRO_PUBLIC' + max_parallel_downloads: 10 + totp_secret: !ENV 'CREODIAS_TOTP_SECRET' 'https://podaac-tools.jpl.nasa.gov/drive/files': username: !ENV 'PODAAC_DRIVE_USERNAME' password: !ENV 'PODAAC_DRIVE_PASSWORD' diff --git a/tests/test_downloaders.py b/tests/test_downloaders.py index 8707413..db7c07a 100644 --- a/tests/test_downloaders.py +++ b/tests/test_downloaders.py @@ -8,10 +8,13 @@ import tempfile import unittest import unittest.mock as mock +from datetime import datetime from pathlib import Path import django.test import oauthlib.oauth2 +import oauthlib.oauth2.rfc6749.errors +import pyotp import requests import requests_oauthlib from geospaas.catalog.managers import LOCAL_FILE_SERVICE @@ -223,7 +226,40 @@ def test_build_oauth2_authentication(self): self.fail(f"oauth2._client does not have the attribute: '{k}'") self.assertEqual(property_value, v, f"oauth2._client.{k} should have the value: '{v}'") - def test_get_oauth2_auth(self): + def test_build_oauth2_authentication_with_totp(self): + """Test that the right TOTP password is generated and used""" + with mock.patch('requests_oauthlib.OAuth2Session.fetch_token') as mock_fetch_token: + now = datetime.now() + downloaders.HTTPDownloader.build_oauth2_authentication( + 'username', 'password', 'token_url', 'client_id', totp_secret='TOTPSECRET') + + mock_fetch_token.assert_called_with( + token_url='token_url', + username='username', + password='password', + client_id='client_id', + totp=pyotp.TOTP('TOTPSECRET').at(now), + ) + + def test_build_oauth2_authentication_totp_retry(self): + """Test that the token retrieval is retried in case the TOTP + authentication fails once.""" + with mock.patch('requests_oauthlib.OAuth2Session.fetch_token') as mock_fetch_token: + mock_fetch_token.side_effect = (oauthlib.oauth2.rfc6749.errors.InvalidGrantError, {}) + oauth2 = downloaders.HTTPDownloader.build_oauth2_authentication( + 'username', 'password', 'token_url', 'client_id', totp_secret='TOTPSECRET') + self.assertIsInstance(oauth2, requests_oauthlib.OAuth2) + + def test_build_oauth2_authentication_totp_error(self): + """Test that the exception is raised in case of persistent TOTP + authentication failure""" + with mock.patch('requests_oauthlib.OAuth2Session.fetch_token') as mock_fetch_token: + mock_fetch_token.side_effect = oauthlib.oauth2.rfc6749.errors.InvalidGrantError + with self.assertRaises(oauthlib.oauth2.rfc6749.errors.InvalidGrantError): + downloaders.HTTPDownloader.build_oauth2_authentication( + 'username', 'password', 'token_url', 'client_id', totp_secret='TOTPSECRET') + + def test_get_oauth2_auth_no_totp(self): """Test getting an OAuth2 authentication from get_auth()""" mock_auth = mock.Mock() with mock.patch( @@ -239,7 +275,28 @@ def test_get_oauth2_auth(self): }), mock_auth ) - mock_build_auth.assert_called_with('username', 'password', 'token_url', 'client_id') + mock_build_auth.assert_called_with('username', 'password', 'token_url', 'client_id', + totp_secret=None) + + def test_get_oauth2_auth_with_totp(self): + """Test getting an OAuth2 authentication from get_auth()""" + mock_auth = mock.Mock() + with mock.patch( + 'geospaas_processing.downloaders.HTTPDownloader.build_oauth2_authentication', + return_value=mock_auth) as mock_build_auth: + self.assertEqual( + downloaders.HTTPDownloader.get_auth({ + 'authentication_type': 'oauth2', + 'username': 'username', + 'password': 'password', + 'token_url': 'token_url', + 'client_id': 'client_id', + 'totp_secret': 'totp_secret', + }), + mock_auth + ) + mock_build_auth.assert_called_with('username', 'password', 'token_url', 'client_id', + totp_secret='totp_secret') def test_get_basic_auth(self): """Test getting a basic authentication from get_auth()""" From 121674680e40a8a4177bf98d1777eb97526bbc2a Mon Sep 17 00:00:00 2001 From: Adrien Perrin Date: Fri, 22 Sep 2023 13:43:28 +0000 Subject: [PATCH 2/3] add pyotp dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index cd2b0cf..84c5313 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ 'nco', 'oauthlib', 'paramiko', + 'pyotp', 'PyYAML', 'requests_oauthlib', 'requests', From da766b1a15ac8d0481988d0f27a77e870c751bdb Mon Sep 17 00:00:00 2001 From: Adrien Perrin Date: Fri, 22 Sep 2023 13:48:39 +0000 Subject: [PATCH 3/3] add pyotp install to docker images --- Dockerfile_cli | 1 + Dockerfile_worker | 1 + 2 files changed, 2 insertions(+) diff --git a/Dockerfile_cli b/Dockerfile_cli index 6325674..711ab84 100644 --- a/Dockerfile_cli +++ b/Dockerfile_cli @@ -3,6 +3,7 @@ FROM ${BASE_IMAGE} AS base RUN pip install --no-cache-dir \ freezegun==1.1.0 \ graypy==2.1.0 \ + 'pyotp' \ redis==3.5.3 \ requests_oauthlib==1.3 diff --git a/Dockerfile_worker b/Dockerfile_worker index c366b52..6c34ce9 100644 --- a/Dockerfile_worker +++ b/Dockerfile_worker @@ -12,6 +12,7 @@ RUN pip install --upgrade --no-cache-dir \ 'importlib-metadata==4.*' \ 'netCDF4>=1.6.0' \ 'paramiko<2.9' \ + 'pyotp' \ 'redis==4.1.*' \ 'requests_oauthlib==1.3.*' \ 'scp==0.14.*' \