From d25e2dd038c2a8828585a4938fb929c316443e37 Mon Sep 17 00:00:00 2001 From: Onur ULUSOY Date: Fri, 24 May 2024 15:07:31 +0000 Subject: [PATCH 1/4] Faster load_module with threading --- upsonic/remote/on_prem.py | 55 ++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/upsonic/remote/on_prem.py b/upsonic/remote/on_prem.py index 95101bf..b0225e8 100644 --- a/upsonic/remote/on_prem.py +++ b/upsonic/remote/on_prem.py @@ -432,6 +432,9 @@ def load_module(self, module_name, version=None): sub_module_name = module_name.replace(".", "_") module_name = sub_module_name + the_threads = [] + + the_all_imports = {} for i in the_all: original_i = i @@ -467,25 +470,41 @@ def load_module(self, module_name, version=None): except: if self.tester: traceback.print_exc() + def gather(): + if version != None: + version_list_response = self.get_version_history(original_i) + version_list = [] + for each_v in version_list_response: + version_list.append(each_v.replace(original_i+":", "")) + + + if version in version_list: + try: + the_all_imports[i] = self.get( + original_i, + version, + pass_python_version_control=True + ) + except: + the_all_imports[i] = self.get(original_i, pass_python_version_control=True) + else: + the_all_imports[i] = self.get(original_i, pass_python_version_control=True) - if version != None: - version_list_response = self.get_version_history(original_i) - version_list = [] - for each_v in version_list_response: - version_list.append(each_v.replace(original_i+":", "")) - - - if version in version_list: - try: - the_all_imports[i] = self.get( - original_i, - version, - pass_python_version_control=True - ) - except: - the_all_imports[i] = self.get(original_i, pass_python_version_control=True) - else: - the_all_imports[i] = self.get(original_i, pass_python_version_control=True) + while len(the_threads) >= self.thread_number: + for each in the_threads: + if not each.is_alive(): + the_threads.remove(each) + time.sleep(0.1) + + the_thread = threading.Thread(target=gather) + the_thread.start() + the_threads.append(the_thread) + + + for each in the_threads: + each.join() + + import types From d03f678710e6760fd2d1fd49c09540f50094f4a6 Mon Sep 17 00:00:00 2001 From: Onur ULUSOY Date: Fri, 24 May 2024 15:13:24 +0000 Subject: [PATCH 2/4] Fix --- upsonic/remote/on_prem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/upsonic/remote/on_prem.py b/upsonic/remote/on_prem.py index b0225e8..ab3672b 100644 --- a/upsonic/remote/on_prem.py +++ b/upsonic/remote/on_prem.py @@ -470,7 +470,7 @@ def load_module(self, module_name, version=None): except: if self.tester: traceback.print_exc() - def gather(): + def gather(the_all_imports): if version != None: version_list_response = self.get_version_history(original_i) version_list = [] @@ -496,7 +496,7 @@ def gather(): the_threads.remove(each) time.sleep(0.1) - the_thread = threading.Thread(target=gather) + the_thread = threading.Thread(target=gather, args=(the_all_imports,)) the_thread.start() the_threads.append(the_thread) @@ -504,7 +504,7 @@ def gather(): for each in the_threads: each.join() - + import types From 15ed263004080fc3b5ca948a292ed7c4288cbe80 Mon Sep 17 00:00:00 2001 From: Onur ULUSOY Date: Fri, 24 May 2024 15:19:29 +0000 Subject: [PATCH 3/4] Fix --- upsonic/remote/on_prem.py | 57 +++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/upsonic/remote/on_prem.py b/upsonic/remote/on_prem.py index ab3672b..bcac02e 100644 --- a/upsonic/remote/on_prem.py +++ b/upsonic/remote/on_prem.py @@ -433,9 +433,9 @@ def load_module(self, module_name, version=None): module_name = sub_module_name the_threads = [] - - the_all_imports = {} + dict_lock = threading.Lock() + for i in the_all: original_i = i if "_upsonic_" in i: @@ -452,62 +452,55 @@ def load_module(self, module_name, version=None): self._log(f"key_version {key_version}") self._log(f"currenly_version {currenly_version}") if key_version[0] == currenly_version[0] and key_version[0] == "3": - if self.tester: + if self.ttester: self._log(f"Versions are same and 3") if key_version[1] != currenly_version[1]: - if self.tester: - self._log("Minor versions are different") - - self._log( - f"[bold orange]Warning: The versions are different, are you sure to continue") - the_input = input("Yes or no (y/n)").lower() - if the_input == "n": - key_version = f"{key_version[0]}.{key_version[1]}" - currenly_version = f"{currenly_version[0]}.{currenly_version[1]}" - return "Python versions is different (Key == " + key_version + " This runtime == " + currenly_version + ")" - if the_input == "y": - version_check_pass = True + if self.tester: + self._log("Minor versions are different") + + self._log(f"[bold orange]Warning: The versions are different, are you sure to continue") + the_input = input("Yes or no (y/n)").lower() + if the_input == "n": + key_version = f"{key_version[0]}.{key_version[1]}" + currenly_version = f"{currenly_version[0]}.{currenly_version[1]}" + return "Python versions is different (Key == " + key_version + " This runtime == " + currenly_version + ")" + if the_input == "y": + version_check_pass = True except: if self.tester: traceback.print_exc() - def gather(the_all_imports): + + def gather(): if version != None: version_list_response = self.get_version_history(original_i) version_list = [] for each_v in version_list_response: - version_list.append(each_v.replace(original_i+":", "")) - + version_list.append(each_v.replace(original_i + ":", "")) if version in version_list: try: - the_all_imports[i] = self.get( - original_i, - version, - pass_python_version_control=True - ) + with dict_lock: + the_all_imports[i] = self.get(original_i, version, pass_python_version_control=True) except: - the_all_imports[i] = self.get(original_i, pass_python_version_control=True) + with dict_lock: + the_all_imports[i] = self.get(original_i, pass_python_version_control=True) else: - the_all_imports[i] = self.get(original_i, pass_python_version_control=True) + with dict_lock: + the_all_imports[i] = self.get(original_i, pass_python_version_control=True) while len(the_threads) >= self.thread_number: for each in the_threads: if not each.is_alive(): the_threads.remove(each) time.sleep(0.1) - - the_thread = threading.Thread(target=gather, args=(the_all_imports,)) + + the_thread = threading.Thread(target=gather) the_thread.start() the_threads.append(the_thread) - for each in the_threads: each.join() - - - import types - def create_module_obj(dictionary): result = {} for key, value in dictionary.items(): From c280daeaf22194a4c6128a13b4750385fb7da3f1 Mon Sep 17 00:00:00 2001 From: Onur ULUSOY Date: Fri, 24 May 2024 15:20:52 +0000 Subject: [PATCH 4/4] Fix --- upsonic/remote/on_prem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upsonic/remote/on_prem.py b/upsonic/remote/on_prem.py index bcac02e..deba066 100644 --- a/upsonic/remote/on_prem.py +++ b/upsonic/remote/on_prem.py @@ -6,7 +6,7 @@ from functools import wraps from hashlib import sha256 - +import types import pickle import os import re