Add TOTP authentication for the HTTP downloader

Dockerfile_cli

@@ -3,6 +3,7 @@ FROM ${BASE_IMAGE} AS base
 RUN pip install --no-cache-dir \
     freezegun==1.1.0 \
     graypy==2.1.0 \
+    'pyotp' \
     redis==3.5.3 \
     requests_oauthlib==1.3
 

Dockerfile_worker

@@ -12,6 +12,7 @@ RUN pip install --upgrade --no-cache-dir \
     'importlib-metadata==4.*' \
     'netCDF4>=1.6.0' \
     'paramiko<2.9' \
+    'pyotp' \
     'redis==4.1.*' \
     'requests_oauthlib==1.3.*' \
     'scp==0.14.*' \

geospaas_processing/downloaders.py

@@ -12,11 +12,13 @@
 import logging
 import os
 import os.path
+import re
 import shutil
 from urllib.parse import urlparse
 
 import oauthlib.oauth2
-import re
+import oauthlib.oauth2.rfc6749.errors
+import pyotp
 import requests
 import requests.utils
 import requests_oauthlib
@@ -161,16 +163,33 @@ class HTTPDownloader(Downloader):
     CHUNK_SIZE = 1024 * 1024
 
     @classmethod
-    def build_oauth2_authentication(cls, username, password, token_url, client_id):
+    def build_oauth2_authentication(cls, username, password, token_url, client_id,
+                                    totp_secret=None):
         """Creates an OAuth2 object usable by `requests` methods"""
-        client = oauthlib.oauth2.LegacyApplicationClient(client_id=client_id)
-        token = requests_oauthlib.OAuth2Session(client=client).fetch_token(
-            token_url=token_url,
-            username=username,
-            password=password,
-            client_id=client_id,
-        )
-        return requests_oauthlib.OAuth2(client_id=client_id, client=client, token=token)
+        # TOTP passwords are valid for 30 seconds, so we retry a few
+        # times in case we get unlucky and the password expires between
+        # the generation of the password and the authentication request
+        retries = 5
+        while retries > 0:
+            client = oauthlib.oauth2.LegacyApplicationClient(client_id=client_id)
+            session_args = {
+                'token_url': token_url,
+                'username': username,
+                'password': password,
+                'client_id': client_id,
+            }
+            if totp_secret:
+                session_args['totp'] = pyotp.TOTP(totp_secret).now()
+
+            try:
+                token = requests_oauthlib.OAuth2Session(client=client).fetch_token(**session_args)
+            except oauthlib.oauth2.rfc6749.errors.InvalidGrantError:
+                retries -= 1
+                if retries  > 0:
+                    continue
+                else:
+                    raise
+            return requests_oauthlib.OAuth2(client_id=client_id, client=client, token=token)
 
     @classmethod
     def get_auth(cls, kwargs):
@@ -185,7 +204,8 @@ def get_auth(cls, kwargs):
                 kwargs['username'],
                 kwargs['password'],
                 kwargs['token_url'],
-                kwargs['client_id']
+                kwargs['client_id'],
+                totp_secret=kwargs.get('totp_secret'),
             )
         else:
             return super().get_auth(kwargs)

geospaas_processing/provider_settings.yml

@@ -31,9 +31,18 @@
   username: !ENV 'CREODIAS_USERNAME'
   password: !ENV 'CREODIAS_PASSWORD'
   authentication_type: 'oauth2'
-  token_url: 'https://identity.cloudferro.com/auth/realms/DIAS/protocol/openid-connect/token'
+  token_url: 'https://identity.cloudferro.com/auth/realms/Creodias-new/protocol/openid-connect/token'
   client_id: 'CLOUDFERRO_PUBLIC'
   max_parallel_downloads: 10
+  totp_secret: !ENV 'CREODIAS_TOTP_SECRET'
+'https://datahub.creodias.eu':
+  username: !ENV 'CREODIAS_USERNAME'
+  password: !ENV 'CREODIAS_PASSWORD'
+  authentication_type: 'oauth2'
+  token_url: 'https://identity.cloudferro.com/auth/realms/Creodias-new/protocol/openid-connect/token'
+  client_id: 'CLOUDFERRO_PUBLIC'
+  max_parallel_downloads: 10
+  totp_secret: !ENV 'CREODIAS_TOTP_SECRET'
 'https://podaac-tools.jpl.nasa.gov/drive/files':
   username: !ENV 'PODAAC_DRIVE_USERNAME'
   password: !ENV 'PODAAC_DRIVE_PASSWORD'

setup.py

@@ -27,6 +27,7 @@
         'nco',
         'oauthlib',
         'paramiko',
+        'pyotp',
         'PyYAML',
         'requests_oauthlib',
         'requests',

tests/test_downloaders.py

@@ -8,10 +8,13 @@
 import tempfile
 import unittest
 import unittest.mock as mock
+from datetime import datetime
 from pathlib import Path
 
 import django.test
 import oauthlib.oauth2
+import oauthlib.oauth2.rfc6749.errors
+import pyotp
 import requests
 import requests_oauthlib
 from geospaas.catalog.managers import LOCAL_FILE_SERVICE
@@ -223,7 +226,40 @@ def test_build_oauth2_authentication(self):
                 self.fail(f"oauth2._client does not have the attribute: '{k}'")
             self.assertEqual(property_value, v, f"oauth2._client.{k} should have the value: '{v}'")
 
-    def test_get_oauth2_auth(self):
+    def test_build_oauth2_authentication_with_totp(self):
+        """Test that the right TOTP password is generated and used"""
+        with mock.patch('requests_oauthlib.OAuth2Session.fetch_token') as mock_fetch_token:
+            now = datetime.now()
+            downloaders.HTTPDownloader.build_oauth2_authentication(
+                'username', 'password', 'token_url', 'client_id', totp_secret='TOTPSECRET')
+
+        mock_fetch_token.assert_called_with(
+            token_url='token_url',
+            username='username',
+            password='password',
+            client_id='client_id',
+            totp=pyotp.TOTP('TOTPSECRET').at(now),
+        )
+
+    def test_build_oauth2_authentication_totp_retry(self):
+        """Test that the token retrieval is retried in case the TOTP
+        authentication fails once."""
+        with mock.patch('requests_oauthlib.OAuth2Session.fetch_token') as mock_fetch_token:
+            mock_fetch_token.side_effect = (oauthlib.oauth2.rfc6749.errors.InvalidGrantError, {})
+            oauth2 = downloaders.HTTPDownloader.build_oauth2_authentication(
+                'username', 'password', 'token_url', 'client_id', totp_secret='TOTPSECRET')
+            self.assertIsInstance(oauth2, requests_oauthlib.OAuth2)
+
+    def test_build_oauth2_authentication_totp_error(self):
+        """Test that the exception is raised in case of persistent TOTP
+        authentication failure"""
+        with mock.patch('requests_oauthlib.OAuth2Session.fetch_token') as mock_fetch_token:
+            mock_fetch_token.side_effect = oauthlib.oauth2.rfc6749.errors.InvalidGrantError
+            with self.assertRaises(oauthlib.oauth2.rfc6749.errors.InvalidGrantError):
+                downloaders.HTTPDownloader.build_oauth2_authentication(
+                    'username', 'password', 'token_url', 'client_id', totp_secret='TOTPSECRET')
+
+    def test_get_oauth2_auth_no_totp(self):
         """Test getting an OAuth2 authentication from get_auth()"""
         mock_auth = mock.Mock()
         with mock.patch(
@@ -239,7 +275,28 @@ def test_get_oauth2_auth(self):
                 }),
                 mock_auth
             )
-        mock_build_auth.assert_called_with('username', 'password', 'token_url', 'client_id')
+        mock_build_auth.assert_called_with('username', 'password', 'token_url', 'client_id',
+                                           totp_secret=None)
+
+    def test_get_oauth2_auth_with_totp(self):
+        """Test getting an OAuth2 authentication from get_auth()"""
+        mock_auth = mock.Mock()
+        with mock.patch(
+                'geospaas_processing.downloaders.HTTPDownloader.build_oauth2_authentication',
+                return_value=mock_auth) as mock_build_auth:
+            self.assertEqual(
+                downloaders.HTTPDownloader.get_auth({
+                    'authentication_type': 'oauth2',
+                    'username': 'username',
+                    'password': 'password',
+                    'token_url': 'token_url',
+                    'client_id': 'client_id',
+                    'totp_secret': 'totp_secret',
+                }),
+                mock_auth
+            )
+        mock_build_auth.assert_called_with('username', 'password', 'token_url', 'client_id',
+                                           totp_secret='totp_secret')
 
     def test_get_basic_auth(self):
         """Test getting a basic authentication from get_auth()"""