Low memory solution, V2.9.4

README.md

@@ -15,7 +15,7 @@
 License
 ======
 
-    Copyright (c) 2018 @T.WKVER </MATRIX>
+    Copyright (c) 2019 @T.WKVER </MATRIX>
     Code by </MATRIX>@Neod Anderjon(LeaderN)
     MIT license read in LICENSE
     Thanks to watch my project
@@ -24,8 +24,8 @@ License
 Update
 ======
 
-    Version: 2.9.3
-    Last Update Time: 20181116pm2108
+    Version: 2.9.4
+    Last Update Time: 20190130pm1423
 
     This python crawler is built to crawl pixiv images
     It have two mode: RankTopN and illustRepoAll 
@@ -135,4 +135,11 @@ Problems that may arise
     which is still being updated, from the V2.8.5 version instead of pycrypto.
     You need to delete the original .aes_crypto_login.ini file, 
     uninstall the pycrypto module from the python environment, 
-    and install the pycryptodome module.
+    and install the pycryptodome module.
+
+    If the system memory is very low, 
+    even the [SYSTEM_MAX_THREADS(setting in dataload.py L42)] threads of the basic settings 
+    can not be created, then the program will be stuck for a period of time 
+    and finally report an error.
+    In order to ensure the successful operation of the program, 
+    please be sure to leave more than 2G free memory.

dataload.py

@@ -11,7 +11,7 @@
 DEVELOPER           = 'Neod Anderjon(LeaderN)'
 LABORATORY          = 'T.WKVER'
 ORGANIZATION        = '</MATRIX>'
-VERSION             = '2.9.3'
+VERSION             = '2.9.4'
 
 # logfile log real-time operation
 base_time = time.time()
@@ -39,7 +39,7 @@ def crawler_logo():
         ' Code by ' + ORGANIZATION + '@' + DEVELOPER)
     logtime_print(log_context)
 
-SYSTEM_MAX_THREADS = 500                # setting system can contain max sub-threads
+SYSTEM_MAX_THREADS = 400                # setting system can contain max sub-threads
 DEFAULT_PURE_PROXYDNS = '8.8.8.8:53'    # default pure dns by Google
 
 def platform_setting():

privmatrix.py

@@ -27,7 +27,7 @@ class PixivAPILib:
     |       ╚═╝     ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝   ╚═════╝╚═╝  ╚═╝╚═╝  ╚═╝ ╚══╝╚══╝ ╚══════╝╚══════╝╚═╝  ╚═╝╚═╝╚═╝╚═╝      |
     |                                                                                                               |
     |       Copyright (c)2018 T.WKVER </MATRIX> Neod Anderjon(LeaderN)                                              |
-    |       Version: 2.9.3 LTE                                                                                      |
+    |       Version: 2.9.4 LTE                                                                                      |
     |       Code by </MATRIX>@Neod Anderjon(LeaderN)                                                                |
     |       PixivCrawlerIII Help Page                                                                               |
     |       1.rtn  ---     RankingTopN, crawl Pixiv daily/weekly/month ranking top artworks                         |
@@ -647,7 +647,15 @@ def create(self):
             self.lock_t.acquire()
             self.queue_t.append(self)
             self.lock_t.release()
-            self.start()        
+            # if the system has insufficient memory
+            # it will not be able to create more threads
+            # this step will fail
+            try:
+                self.start()        
+            except Exception as e:
+                log_context = "Error Type: " + str(e)
+                PixivAPILib.logprowork(log_context, self.logpath)
+                exit
 
     def timer_decorator(origin_func):
         """Timer decorator