Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Faster load_module with threading #15

Closed
wants to merge 4 commits into from
Closed

Faster load_module with threading #15

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d25e2dd
Faster load_module with threading
onuratakan May 24, 2024
d03f678
Fix
onuratakan May 24, 2024
15ed263
Fix
onuratakan May 24, 2024
c280dae
Fix
onuratakan May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 44 additions & 32 deletions upsonic/remote/on_prem.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from functools import wraps
from hashlib import sha256


import types
import pickle
import os
import re
Expand Down Expand Up @@ -432,7 +432,10 @@ def load_module(self, module_name, version=None):
sub_module_name = module_name.replace(".", "_")
module_name = sub_module_name

the_threads = []
the_all_imports = {}
dict_lock = threading.Lock()

for i in the_all:
original_i = i
if "_upsonic_" in i:
Expand All @@ -449,45 +452,54 @@ def load_module(self, module_name, version=None):
self._log(f"key_version {key_version}")
self._log(f"currenly_version {currenly_version}")
if key_version[0] == currenly_version[0] and key_version[0] == "3":
if self.tester:
if self.ttester:
self._log(f"Versions are same and 3")
if key_version[1] != currenly_version[1]:
if self.tester:
self._log("Minor versions are different")

self._log(
f"[bold orange]Warning: The versions are different, are you sure to continue")
the_input = input("Yes or no (y/n)").lower()
if the_input == "n":
key_version = f"{key_version[0]}.{key_version[1]}"
currenly_version = f"{currenly_version[0]}.{currenly_version[1]}"
return "Python versions is different (Key == " + key_version + " This runtime == " + currenly_version + ")"
if the_input == "y":
version_check_pass = True
if self.tester:
self._log("Minor versions are different")

self._log(f"[bold orange]Warning: The versions are different, are you sure to continue")
the_input = input("Yes or no (y/n)").lower()
if the_input == "n":
key_version = f"{key_version[0]}.{key_version[1]}"
currenly_version = f"{currenly_version[0]}.{currenly_version[1]}"
return "Python versions is different (Key == " + key_version + " This runtime == " + currenly_version + ")"
if the_input == "y":
version_check_pass = True
except:
if self.tester:
traceback.print_exc()

if version != None:
version_list_response = self.get_version_history(original_i)
version_list = []
for each_v in version_list_response:
version_list.append(each_v.replace(original_i+":", ""))


if version in version_list:
try:
the_all_imports[i] = self.get(
original_i,
version,
pass_python_version_control=True
)
except:
def gather():
if version != None:
version_list_response = self.get_version_history(original_i)
version_list = []
for each_v in version_list_response:
version_list.append(each_v.replace(original_i + ":", ""))

if version in version_list:
try:
with dict_lock:
the_all_imports[i] = self.get(original_i, version, pass_python_version_control=True)
except:
with dict_lock:
the_all_imports[i] = self.get(original_i, pass_python_version_control=True)
else:
with dict_lock:
the_all_imports[i] = self.get(original_i, pass_python_version_control=True)
else:
the_all_imports[i] = self.get(original_i, pass_python_version_control=True)

import types
while len(the_threads) >= self.thread_number:
for each in the_threads:
if not each.is_alive():
the_threads.remove(each)
time.sleep(0.1)

the_thread = threading.Thread(target=gather)
the_thread.start()
the_threads.append(the_thread)

for each in the_threads:
each.join()

def create_module_obj(dictionary):
result = {}
Expand Down
Loading