Add support for resolving urls

docs/CHANGELOG.rst

@@ -7,6 +7,15 @@ file.
 This project adheres to `Semantic Versioning <http://semver.org/>`_.
 
 
+==========
+Unreleased
+==========
+
+Added
+-----
+- Add support for resolving urls
+
+
 ==================
 3.1.1 - 2021-07-21
 ==================

resolwe_runtime_utils.py

@@ -27,13 +27,13 @@
 import socket
 import subprocess
 import tarfile
-import zlib
 import time
+import zlib
+from contextlib import suppress
 from enum import Enum
 from itertools import zip_longest
 from pathlib import Path
 
-
 # Socket constants. The timeout is set to infinite.
 SOCKET_TIMEOUT = None
 if "SOCKET_TIMEOUT" in os.environ:
@@ -86,7 +86,7 @@ def wrapper_retry(*args, **kwargs):
                     time.sleep(sleep)
                     return func(*args, **kwargs)
                 except retry_exceptions as err:
-                    sleep = min(max_sleep, min_sleep * (2 ** retry))
+                    sleep = min(max_sleep, min_sleep * (2**retry))
                     last_error = err
             raise last_error
 
@@ -186,6 +186,7 @@ def _check_response(response):
 
     finally:
         sock.close()
+        return response
 
 
 def collect_entry(entry, references):
@@ -768,6 +769,9 @@ def importUncompressed():
     # Large file download from Google Drive requires cookie and token.
     try:
         response = None
+        # Resolve url.
+        src = send_message(command("resolve_url", src))["data"]
+
         if re.match(
             r'^https://drive.google.com/[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$',
             src,
@@ -790,6 +794,7 @@ def importUncompressed():
             src,
         ):
             response = requests.get(src, stream=True)
+
     except requests.exceptions.ConnectionError:
         raise requests.exceptions.ConnectionError("Could not connect to {}".format(src))
 

test_resolwe_runtime_utils.py

@@ -12,56 +12,56 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=missing-docstring
-from io import StringIO
 import json
 import os
 import shutil
 import socket
 import sys
 import tempfile
 import time
+
+# pylint: disable=missing-docstring
+from io import StringIO
 from pathlib import Path
 from threading import Thread
 from unittest import TestCase
 from unittest.mock import patch
 
-import responses
 import requests
+import responses
 
 from resolwe_runtime_utils import (
+    ImportedFormat,
+    _re_annotate_entity_main,
+    _re_checkrc_main,
+    _re_error_main,
+    _re_export_main,
+    _re_info_main,
+    _re_progress_main,
+    _re_save_dir_list_main,
+    _re_save_dir_main,
+    _re_save_file_list_main,
+    _re_save_file_main,
+    _re_save_list_main,
+    _re_save_main,
+    _re_warning_main,
     annotate_entity,
-    send_message,
-    save,
+    checkrc,
+    error,
     export_file,
     import_file,
-    ImportedFormat,
-    save_list,
-    save_file,
-    save_file_list,
+    info,
+    progress,
+    save,
     save_dir,
     save_dir_list,
-    info,
+    save_file,
+    save_file_list,
+    save_list,
+    send_message,
     warning,
-    error,
-    progress,
-    checkrc,
-    _re_annotate_entity_main,
-    _re_save_main,
-    _re_export_main,
-    _re_save_list_main,
-    _re_save_file_main,
-    _re_save_file_list_main,
-    _re_save_dir_main,
-    _re_save_dir_list_main,
-    _re_info_main,
-    _re_warning_main,
-    _re_error_main,
-    _re_progress_main,
-    _re_checkrc_main,
 )
 
-
 TEST_ROOT = os.path.abspath(os.path.dirname(__file__))
 
 
@@ -683,15 +683,18 @@ def assertImportFile(self, src, dst, returned_name):
         assert not os.path.exists(returned_name), "file should not exist"
         os.remove(returned_name + '.gz')
 
-    def test_uncompressed(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_uncompressed(self, send_mock):
         self.assertImportFile(
             'some file.1.txt', 'test uncompressed.txt', 'test uncompressed.txt'
         )
 
-    def test_gz(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_gz(self, send_mock):
         self.assertImportFile('some file.1.txt.gz', 'test gz.txt.gz', 'test gz.txt')
 
-    def test_7z(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_7z(self, send_mock):
         self.assertImportFile('some file.1.txt.zip', 'test 7z.txt.zip', 'test 7z.txt')
 
         file_name = import_file(self._file('some folder.tar.gz'), 'some folder.tar.gz')
@@ -726,11 +729,13 @@ def test_7z(self):
         assert not os.path.isdir('some folder 1'), "directory should not exist"
         assert os.path.exists('some folder 1.tar.gz'), "file not found"
 
-    def test_7z_corrupted(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_7z_corrupted(self, send_mock):
         with self.assertRaises(ValueError, msg='failed to extract file: corrupted.zip'):
             import_file(self._file('corrupted.zip'), 'corrupted.zip')
 
-    def test_gz_corrupted(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_gz_corrupted(self, send_mock):
         with self.assertRaises(
             ValueError, msg='invalid gzip file format: corrupted.gz'
         ):
@@ -744,7 +749,8 @@ def test_gz_corrupted(self):
             )
 
     @responses.activate
-    def test_uncompressed_url(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_uncompressed_url(self, send_mock):
         responses.add(
             responses.GET, 'https://testurl/someslug', status=200, body='some text'
         )
@@ -754,7 +760,8 @@ def test_uncompressed_url(self):
         assert os.path.exists('test uncompressed.txt.gz'), "file not found"
 
     @responses.activate
-    def test_gz_url(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_gz_url(self, send_mock):
         # Return gzipped file
         responses.add(
             responses.GET,
@@ -771,7 +778,8 @@ def test_gz_url(self):
         assert os.path.exists('test uncompressed.txt.gz'), "file not found"
 
     @responses.activate
-    def test_7z_url(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_7z_url(self, send_mock):
         # Return zipped file
         responses.add(
             responses.GET,
@@ -792,7 +800,8 @@ def test_7z_url(self):
         assert os.path.exists('test uncompressed.txt'), "file not found"
         assert os.path.exists('test uncompressed.txt.gz'), "file not found"
 
-    def test_invalid_url(self):
+    @patch('resolwe_runtime_utils.send_message', side_effect=lambda x: x)
+    def test_invalid_url(self, send_mock):
         with self.assertRaises(requests.exceptions.ConnectionError):
             import_file('http://testurl/someslug', 'test uncompressed.txt.zip')