Merge pull request #5427 from devos50/devel_update

Merge release-7.5 into devel

build/mac/makedist_macos.sh

@@ -14,9 +14,6 @@ export RESOURCES=build/mac/resources
 
 # ----- Build
 PI=pyinstaller
-if [ "$(which pyinstaller-2.7)" != "" ]; then
-	PI=pyinstaller-2.7
-fi
 $PI tribler.spec
 
 mkdir -p dist/installdir

src/tribler-core/tribler_core/modules/libtorrent/tests/test_torrent_def.py

@@ -267,3 +267,14 @@ def test_get_index_no_metainfo(self):
         """
         t = TorrentDef()
         self.assertRaises(ValueError, t.get_index_of_file_in_files, u'?')
+
+    def test_get_name_as_unicode(self):
+        name_bytes = b'\xe8\xaf\xad\xe8\xa8\x80\xe5\xa4\x84\xe7\x90\x86'
+        name_unicode = name_bytes.decode()
+        t = TorrentDef()
+        t.metainfo = {b'info': {b'name.utf-8': name_bytes}}
+        self.assertEqual(t.get_name_as_unicode(), name_unicode)
+        t.metainfo = {b'info': {b'name': name_bytes}}
+        self.assertEqual(t.get_name_as_unicode(), name_unicode)
+        t.metainfo = {b'info': {b'name': b'test\xff' + name_bytes}}
+        self.assertEqual(t.get_name_as_unicode(), 'test?????????????')

src/tribler-core/tribler_core/modules/libtorrent/torrentdef.py

@@ -309,11 +309,10 @@ def get_name_as_unicode(self):
             try:
                 def filter_characters(name):
                     def filter_character(char):
-                        if 0 < ord(char) < 128:
+                        if 0 < char < 128:
                             return chr(char)
-                        else:
-                            self._logger.debug("Bad character filter %s, isalnum? %s", bytes(char), char.isalnum())
-                            return u"?"
+                        self._logger.debug("Bad character %s", bytes(char))
+                        return u"?"
                     return u"".join([filter_character(char) for char in name])
                 return str(filter_characters(self.metainfo[b"info"][b"name"]))
             except UnicodeError:

src/tribler-core/tribler_core/modules/metadata_store/gigachannel_manager.py

@@ -1,5 +1,5 @@
 import asyncio
-from asyncio import get_event_loop, wait_for
+from asyncio import CancelledError, get_event_loop, wait_for
 
 from ipv8.database import database_blob
 from ipv8.taskmanager import TaskManager, task
@@ -271,8 +271,12 @@ async def download_channel(self, channel):
         tdef = TorrentDef(metainfo=metainfo)
 
         download = self.session.dlmgr.start_download(tdef=tdef, config=dcfg, hidden=True)
-        await download.future_finished
-        self.channels_processing_queue[channel.infohash] = (PROCESS_CHANNEL_DIR, channel)
+        try:
+            await download.future_finished
+        except CancelledError:
+            pass
+        else:
+            self.channels_processing_queue[channel.infohash] = (PROCESS_CHANNEL_DIR, channel)
         return download
 
     async def process_channel_dir_threaded(self, channel):

src/tribler-core/tribler_core/modules/tests/test_tracker_manager.py

@@ -65,7 +65,7 @@ def test_get_tracker_for_check(self):
         self.assertFalse(self.tracker_manager.get_next_tracker_for_auto_check())
 
         self.tracker_manager.add_tracker("http://test1.com:80/announce")
-        self.assertEqual('http://test1.com/announce', self.tracker_manager.get_next_tracker_for_auto_check())
+        self.assertEqual('http://test1.com/announce', self.tracker_manager.get_next_tracker_for_auto_check().url)
 
     def test_get_tracker_for_check_blacklist(self):
         """

src/tribler-core/tribler_core/modules/torrent_checker/test_torrentchecker.py

@@ -8,6 +8,7 @@
 from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker
 from tribler_core.modules.torrent_checker.torrentchecker_session import HttpTrackerSession, UdpSocketManager
 from tribler_core.modules.tracker_manager import TrackerManager
+from tribler_core.tests.tools.base_test import MockObject
 from tribler_core.tests.tools.test_as_server import TestAsServer
 from tribler_core.tests.tools.tools import timeout
 from tribler_core.utilities.unicode import hexlify
@@ -74,7 +75,7 @@ async def test_health_check_blacklisted_trackers(self):
         with db_session:
             tracker = self.session.mds.TrackerState(url="http://localhost/tracker")
             self.session.mds.TorrentState(infohash=b'a' * 20, seeders=5, leechers=10, trackers={tracker},
-                                             last_check=int(time.time()))
+                                          last_check=int(time.time()))
 
         self.session.tracker_manager.blacklist.append("http://localhost/tracker")
         result = await self.torrent_checker.check_torrent_health(b'a' * 20)
@@ -89,7 +90,7 @@ async def test_health_check_cached(self):
         with db_session:
             tracker = self.session.mds.TrackerState(url="http://localhost/tracker")
             self.session.mds.TorrentState(infohash=b'a' * 20, seeders=5, leechers=10, trackers={tracker},
-                                             last_check=int(time.time()))
+                                          last_check=int(time.time()))
 
         result = await self.torrent_checker.check_torrent_health(b'a' * 20)
         self.assertTrue('db' in result)
@@ -98,18 +99,48 @@ async def test_health_check_cached(self):
 
     @timeout(10)
     async def test_task_select_no_tracker(self):
-        await self.torrent_checker.check_random_tracker()
+        """
+        Test whether we are not checking a random tracker if there are no trackers in the database.
+        """
+        result = await self.torrent_checker.check_random_tracker()
+        self.assertFalse(result)
+
+    @timeout(10)
+    async def test_check_random_tracker_shutdown(self):
+        """
+        Test whether we are not performing a tracker check if we are shutting down.
+        """
+        await self.torrent_checker.shutdown()
+        result = await self.torrent_checker.check_random_tracker()
+        self.assertFalse(result)
+
+    @timeout(10)
+    async def test_check_random_tracker_not_alive(self):
+        """
+        Test whether we correctly update the tracker state when the number of failures is too large.
+        """
+        with db_session:
+            self.session.mds.TrackerState(url="http://localhost/tracker", failures=1000, alive=True)
 
+        result = await self.torrent_checker.check_random_tracker()
+        self.assertFalse(result)
+
+        with db_session:
+            tracker = self.session.tracker_manager.tracker_store.get()
+            self.assertFalse(tracker.alive)
+
+    @timeout(10)
     async def test_task_select_tracker(self):
         with db_session:
             tracker = self.session.mds.TrackerState(url="http://localhost/tracker")
             self.session.mds.TorrentState(infohash=b'a' * 20, seeders=5, leechers=10, trackers={tracker})
 
-        controlled_session = HttpTrackerSession(None, None, None, None)
+        controlled_session = HttpTrackerSession("127.0.0.1", ("localhost", 8475), "/announce", 5)
         controlled_session.connect_to_tracker = lambda: succeed(None)
 
         self.torrent_checker._create_session_for_request = lambda *args, **kwargs: controlled_session
-        await self.torrent_checker.check_random_tracker()
+        result = await self.torrent_checker.check_random_tracker()
+        self.assertFalse(result)
 
         self.assertEqual(len(controlled_session.infohash_list), 1)
 
@@ -121,8 +152,9 @@ async def test_tracker_test_error_resolve(self):
         with db_session:
             tracker = self.session.mds.TrackerState(url="http://localhost/tracker")
             self.session.mds.TorrentState(infohash=b'a' * 20, seeders=5, leechers=10, trackers={tracker},
-                                             last_check=int(time.time()))
-        await self.torrent_checker.check_random_tracker()
+                                          last_check=int(time.time()))
+        result = await self.torrent_checker.check_random_tracker()
+        self.assertFalse(result)
 
         # Verify whether we successfully cleaned up the session after an error
         self.assertEqual(len(self.torrent_checker._session_list), 1)
@@ -133,25 +165,31 @@ async def test_tracker_no_infohashes(self):
         Test the check of a tracker without associated torrents
         """
         self.session.tracker_manager.add_tracker('http://trackertest.com:80/announce')
-        await self.torrent_checker.check_random_tracker()
+        result = await self.torrent_checker.check_random_tracker()
+        self.assertFalse(result)
 
     def test_get_valid_next_tracker_for_auto_check(self):
-        """ Test if only valid tracker url is used for auto check """
-        test_tracker_list = ["http://anno nce.torrentsmd.com:8080/announce",
-                             "http://announce.torrentsmd.com:8080/announce"]
+        """
+        Test if only valid tracker url are used for auto check
+        """
+        mock_tracker_state_invalid = MockObject()
+        mock_tracker_state_invalid.url = "http://anno nce.torrentsmd.com:8080/announce"
+        mock_tracker_state_valid = MockObject()
+        mock_tracker_state_valid.url = "http://announce.torrentsmd.com:8080/announce"
+        tracker_states = [mock_tracker_state_invalid, mock_tracker_state_valid]
 
         def get_next_tracker_for_auto_check():
-            return test_tracker_list[0] if test_tracker_list else None
+            return tracker_states[0] if tracker_states else None
 
-        def remove_tracker(tracker_url):
-            test_tracker_list.remove(tracker_url)
+        def remove_tracker(_):
+            tracker_states.remove(mock_tracker_state_invalid)
 
         self.torrent_checker.get_next_tracker_for_auto_check = get_next_tracker_for_auto_check
         self.torrent_checker.remove_tracker = remove_tracker
 
-        next_tracker_url = self.torrent_checker.get_valid_next_tracker_for_auto_check()
-        self.assertEqual(len(test_tracker_list), 1)
-        self.assertEqual(next_tracker_url, "http://announce.torrentsmd.com:8080/announce")
+        next_tracker = self.torrent_checker.get_valid_next_tracker_for_auto_check()
+        self.assertEqual(len(tracker_states), 1)
+        self.assertEqual(next_tracker.url, "http://announce.torrentsmd.com:8080/announce")
 
     def test_on_health_check_completed(self):
         tracker1 = 'udp://localhost:2801'

src/tribler-core/tribler_core/modules/torrent_checker/torrent_checker.py

@@ -86,52 +86,55 @@ async def check_random_tracker(self):
         """
         Calling this method will fetch a random tracker from the database, select some torrents that have this
         tracker, and perform a request to these trackers.
+        Return whether the check was successful.
         """
-        tracker_url = self.get_valid_next_tracker_for_auto_check()
-        if tracker_url is None:
+        if self._should_stop:
+            self._logger.warning("Not performing tracker check since we are shutting down")
+            return False
+
+        tracker = self.get_valid_next_tracker_for_auto_check()
+        if tracker is None:
             self._logger.warning(u"No tracker to select from, skip")
-            return
+            return False
 
-        self._logger.debug(u"Start selecting torrents on tracker %s.", tracker_url)
+        self._logger.debug(u"Start selecting torrents on tracker %s.", tracker.url)
 
         # get the torrents that should be checked
         with db_session:
-            tracker = self.tribler_session.mds.TrackerState.get(url=tracker_url)
-            if not tracker:
-                return
             dynamic_interval = TORRENT_CHECK_RETRY_INTERVAL * (2 ** tracker.failures)
             # FIXME: this is a really dumb fix for update_tracker_info not being called in some cases
             if tracker.failures >= MAX_TRACKER_FAILURES:
-                tracker.alive = False
-                return
+                self.update_tracker_info(tracker.url, False)
+                return False
             torrents = select(ts for ts in tracker.torrents if ts.last_check + dynamic_interval < int(time.time()))
             infohashes = [t.infohash for t in torrents[:MAX_TORRENTS_CHECKED_PER_SESSION]]
 
         if len(infohashes) == 0:
             # We have no torrent to recheck for this tracker. Still update the last_check for this tracker.
-            self._logger.info("No torrent to check for tracker %s", tracker_url)
-            self.update_tracker_info(tracker_url, True)
-            return
+            self._logger.info("No torrent to check for tracker %s", tracker.url)
+            self.update_tracker_info(tracker.url, True)
+            return False
 
         try:
-            session = self._create_session_for_request(tracker_url, timeout=30)
+            session = self._create_session_for_request(tracker.url, timeout=30)
         except MalformedTrackerURLException as e:
             # Remove the tracker from the database
-            self.remove_tracker(tracker_url)
+            self.remove_tracker(tracker.url)
             self._logger.error(e)
-            return
+            return False
 
         # We shuffle the list so that different infohashes are checked on subsequent scrape requests if the total
         # number of infohashes exceeds the maximum number of infohashes we check.
         random.shuffle(infohashes)
         for infohash in infohashes:
             session.add_infohash(infohash)
 
-        self._logger.info(u"Selected %d new torrents to check on tracker: %s", len(infohashes), tracker_url)
+        self._logger.info(u"Selected %d new torrents to check on tracker: %s", len(infohashes), tracker.url)
         try:
             await self.connect_to_tracker(session)
+            return True
         except:
-            pass
+            return False
 
     async def connect_to_tracker(self, session):
         try:
@@ -174,20 +177,20 @@ def check_random_torrent(self):
         return [bytes(random_torrent.infohash)]
 
     def get_valid_next_tracker_for_auto_check(self):
-        tracker_url = self.get_next_tracker_for_auto_check()
-        while tracker_url and not is_valid_url(tracker_url):
-            self.remove_tracker(tracker_url)
-            tracker_url = self.get_next_tracker_for_auto_check()
-        return tracker_url
+        tracker = self.get_next_tracker_for_auto_check()
+        while tracker and not is_valid_url(tracker.url):
+            self.remove_tracker(tracker.url)
+            tracker = self.get_next_tracker_for_auto_check()
+        return tracker
 
     def get_next_tracker_for_auto_check(self):
         return self.tribler_session.tracker_manager.get_next_tracker_for_auto_check()
 
     def remove_tracker(self, tracker_url):
         self.tribler_session.tracker_manager.remove_tracker(tracker_url)
 
-    def update_tracker_info(self, tracker_url, value):
-        self.tribler_session.tracker_manager.update_tracker_info(tracker_url, value)
+    def update_tracker_info(self, tracker_url, is_successful):
+        self.tribler_session.tracker_manager.update_tracker_info(tracker_url, is_successful)
 
     def is_blacklisted_tracker(self, tracker_url):
         return tracker_url in self.tribler_session.tracker_manager.blacklist

src/tribler-core/tribler_core/modules/tracker_manager.py

@@ -134,4 +134,4 @@ def get_next_tracker_for_auto_check(self):
 
         if not tracker:
             return None
-        return tracker[0].url
+        return tracker[0]

src/tribler-core/tribler_core/modules/tunnel/test_full_session/test_tunnel_base.py

@@ -201,6 +201,7 @@ def start_anon_download(self, hops=1):
         """
         Start an anonymous download in the main Tribler session.
         """
+        self.session.config.set_libtorrent_dht_readiness_timeout(0)
         dscfg = DownloadConfig()
         dscfg.set_dest_dir(self.getDestDir())
         dscfg.set_hops(hops)

src/tribler-core/tribler_core/restapi/debug_endpoint.py

@@ -339,4 +339,4 @@ async def start_profiler(self, request):
     )
     async def stop_profiler(self, request):
         file_path = self.session.resource_monitor.stop_profiler()
-        return RESTResponse({"success": True, "profiler_file": file_path})
+        return RESTResponse({"success": True, "profiler_file": str(file_path)})

src/tribler-core/tribler_core/tests/test_session.py

@@ -1,4 +1,4 @@
-from asyncio import get_event_loop
+from asyncio import get_event_loop, sleep
 from binascii import unhexlify
 from unittest.mock import Mock
 
@@ -63,7 +63,7 @@ def function_that_triggers_exception():
             self.assertTrue("function_that_triggers_exception" in m.call_args[0][0])
             self.assertTrue("foobar" in m.call_args[0][0])
 
-    def test_error_observer_ignored_error(self):
+    async def test_error_observer_ignored_error(self):
         """
         Testing whether some errors are ignored (like socket errors)
         """
@@ -78,19 +78,25 @@ def gen_except():
                 raise exception
 
             get_event_loop().call_soon(gen_except)
-            self.loop._run_once()
 
         exceptions_list = [exc_class(errno, "exc message") for exc_class, errno in IGNORED_ERRORS.keys()]
         exceptions_list.append(RuntimeError(0, "invalid info-hash"))
 
         for exception in exceptions_list:
             generate_exception_on_reactor(exception)
+
+        # Even though we could have used _run_once instead of a sleep, it seems that _run_once does not always
+        # immediately clean the reactor, leading to a possibility that the test starts to shut down before the exception
+        # is raised.
+        await sleep(0.05)
+
         self.session.api_manager.get_endpoint('state').on_tribler_exception.assert_not_called()
         self.session.api_manager.get_endpoint('events').on_tribler_exception.assert_not_called()
 
         # This is a "canary" to test that we can handle true exceptions
         get_event_loop().call_soon(getaddrinfo, "dfdfddfd23424fdfdf", 2323)
-        self.loop._run_once()
+
+        await sleep(0.05)
 
         self.session.api_manager.get_endpoint('state').on_tribler_exception.assert_not_called()
         self.session.api_manager.get_endpoint('events').on_tribler_exception.assert_not_called()
@@ -100,7 +106,7 @@ def real_raise():
             raise Exception()
 
         get_event_loop().call_soon(real_raise)
-        self.loop._run_once()
+        await sleep(0.05)
         self.session.api_manager.get_endpoint('state').on_tribler_exception.assert_called_once()
         self.session.api_manager.get_endpoint('events').on_tribler_exception.assert_called_once()
 

src/tribler-gui/tribler_gui/core_manager.py

@@ -43,9 +43,10 @@ def __init__(self, api_port, api_key):
 
     def on_core_read_ready(self):
         raw_output = bytes(self.core_process.readAll())
+        decoded_output = raw_output.decode(errors="replace")
         if b'Traceback' in raw_output:
-            self.core_traceback = raw_output.decode()
-        print(raw_output.decode().strip())
+            self.core_traceback = decoded_output
+        print(decoded_output.strip())
 
     def on_core_finished(self, exit_code, exit_status):
         if self.shutting_down and self.should_stop_on_shutdown: