diff --git a/upsonic/remote/on_prem.py b/upsonic/remote/on_prem.py index 95101bf..deba066 100644 --- a/upsonic/remote/on_prem.py +++ b/upsonic/remote/on_prem.py @@ -6,7 +6,7 @@ from functools import wraps from hashlib import sha256 - +import types import pickle import os import re @@ -432,7 +432,10 @@ def load_module(self, module_name, version=None): sub_module_name = module_name.replace(".", "_") module_name = sub_module_name + the_threads = [] the_all_imports = {} + dict_lock = threading.Lock() + for i in the_all: original_i = i if "_upsonic_" in i: @@ -449,45 +452,54 @@ def load_module(self, module_name, version=None): self._log(f"key_version {key_version}") self._log(f"currenly_version {currenly_version}") if key_version[0] == currenly_version[0] and key_version[0] == "3": - if self.tester: + if self.ttester: self._log(f"Versions are same and 3") if key_version[1] != currenly_version[1]: - if self.tester: - self._log("Minor versions are different") - - self._log( - f"[bold orange]Warning: The versions are different, are you sure to continue") - the_input = input("Yes or no (y/n)").lower() - if the_input == "n": - key_version = f"{key_version[0]}.{key_version[1]}" - currenly_version = f"{currenly_version[0]}.{currenly_version[1]}" - return "Python versions is different (Key == " + key_version + " This runtime == " + currenly_version + ")" - if the_input == "y": - version_check_pass = True + if self.tester: + self._log("Minor versions are different") + + self._log(f"[bold orange]Warning: The versions are different, are you sure to continue") + the_input = input("Yes or no (y/n)").lower() + if the_input == "n": + key_version = f"{key_version[0]}.{key_version[1]}" + currenly_version = f"{currenly_version[0]}.{currenly_version[1]}" + return "Python versions is different (Key == " + key_version + " This runtime == " + currenly_version + ")" + if the_input == "y": + version_check_pass = True except: if self.tester: traceback.print_exc() - if version != None: - version_list_response = self.get_version_history(original_i) - version_list = [] - for each_v in version_list_response: - version_list.append(each_v.replace(original_i+":", "")) - - - if version in version_list: - try: - the_all_imports[i] = self.get( - original_i, - version, - pass_python_version_control=True - ) - except: + def gather(): + if version != None: + version_list_response = self.get_version_history(original_i) + version_list = [] + for each_v in version_list_response: + version_list.append(each_v.replace(original_i + ":", "")) + + if version in version_list: + try: + with dict_lock: + the_all_imports[i] = self.get(original_i, version, pass_python_version_control=True) + except: + with dict_lock: + the_all_imports[i] = self.get(original_i, pass_python_version_control=True) + else: + with dict_lock: the_all_imports[i] = self.get(original_i, pass_python_version_control=True) - else: - the_all_imports[i] = self.get(original_i, pass_python_version_control=True) - import types + while len(the_threads) >= self.thread_number: + for each in the_threads: + if not each.is_alive(): + the_threads.remove(each) + time.sleep(0.1) + + the_thread = threading.Thread(target=gather) + the_thread.start() + the_threads.append(the_thread) + + for each in the_threads: + each.join() def create_module_obj(dictionary): result = {}