From 749220ca1c8f0f73cb5a2f9b0bc4eea8d90f133f Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sat, 31 Jul 2021 00:37:54 +0100 Subject: [PATCH 01/35] added couple simple pickle helper methods --- osbot_utils/utils/Files.py | 18 ++++++++++++++++++ osbot_utils/utils/Misc.py | 1 + tests/utils/test_Files.py | 7 +++++++ 3 files changed, 26 insertions(+) diff --git a/osbot_utils/utils/Files.py b/osbot_utils/utils/Files.py index cb6a96d0..64f5f21f 100644 --- a/osbot_utils/utils/Files.py +++ b/osbot_utils/utils/Files.py @@ -1,6 +1,7 @@ import gzip import os import glob +import pickle import shutil import tempfile import zipfile @@ -176,6 +177,21 @@ def parent_folder(path): def parent_folder_combine(file, path): return Files.path_combine(os.path.dirname(file),path) + @staticmethod + def pickle_save_to_file(object_to_save, path=None): + path = path or temp_file(extension=".pickle") + file_to_store = open(path, "wb") + pickle.dump(object_to_save, file_to_store) + file_to_store.close() + return path + + @staticmethod + def pickle_load_from_file(path=None): + file_to_read = open(path, "rb") + loaded_object = pickle.load(file_to_read) + file_to_read.close() + return loaded_object + @staticmethod def save(contents, path=None, extension=None): path = path or temp_file(extension=extension) @@ -338,6 +354,8 @@ def zip_files(base_folder, file_pattern="*.*", target_file=None): path_current = Files.current_folder parent_folder = Files.parent_folder parent_folder_combine = Files.parent_folder_combine +pickle_load_from_file = Files.pickle_load_from_file +pickle_save_to_file = Files.pickle_save_to_file save_bytes_as_file = Files.save_bytes_as_file save_string_as_file = Files.save diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 37936103..89769763 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -261,6 +261,7 @@ def lower(target : str): return target.lower() return "" + def str_index(target:str, source:str): try: return target.index(source) diff --git a/tests/utils/test_Files.py b/tests/utils/test_Files.py index 0acab53b..8b998766 100644 --- a/tests/utils/test_Files.py +++ b/tests/utils/test_Files.py @@ -199,6 +199,13 @@ def test_folder_zip(self): def test_path_combine(self): assert path_combine('a', 'b') == f"{path_current()}/a/b" # todo: add more use cases + + def test_pickle_save_to_file__pickle_load_from_file(self): + an_object = {"answer" : 42 } + pickled_file = Files.pickle_save_to_file(an_object) + pickle_data = Files.pickle_load_from_file(pickled_file) + assert pickle_data == an_object + def test_save_string_as_file(self): data = random_string() temp_file = save_string_as_file(data) From 2fdcdddd26048df62faa370477b5725a6a399969 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sat, 31 Jul 2021 13:02:53 +0100 Subject: [PATCH 02/35] removed some chars from the random_password method --- osbot_utils/utils/Misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 89769763..25829d0e 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -340,8 +340,8 @@ def random_password(length=24, prefix=''): string.punctuation + string.digits , k=length)) - # replace these chars with _ (to make prevent errors in command prompts) - items = ['"', '\'', '`','\\','}'] + # replace these chars with _ (to make prevent errors in command prompts and urls) + items = ['"', '\'', '`','\\','/','}','?','#',';',':'] for item in items: password = password.replace(item, '_') return password From f3a66a18515ae996a5156794de9a53c93f482e9f Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sat, 31 Jul 2021 15:36:59 +0100 Subject: [PATCH 03/35] added Misc.unique method --- osbot_utils/utils/Misc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 25829d0e..a4110148 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -409,6 +409,9 @@ def trim(target): def under_debugger(): return 'pydevd' in sys.modules +def unique(target): + return list_set(target) + def upper(target : str): if target: return target.upper() From a16085f8769ca7cc48822cb1570ee0870c47292d Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 1 Aug 2021 02:46:38 +0100 Subject: [PATCH 04/35] added url_encode and url_decode methods --- osbot_utils/utils/Misc.py | 9 +++++++++ tests/utils/test_Misc.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index a4110148..f4b77239 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -12,6 +12,7 @@ from datetime import datetime from secrets import token_bytes from time import sleep +from urllib.parse import urlencode, quote_plus, unquote_plus from dotenv import load_dotenv @@ -412,6 +413,14 @@ def under_debugger(): def unique(target): return list_set(target) +def url_encode(data): + if type(data) is str: + return quote_plus(data) + +def url_decode(data): + if type(data) is str: + return unquote_plus(data) + def upper(target : str): if target: return target.upper() diff --git a/tests/utils/test_Misc.py b/tests/utils/test_Misc.py index ea61f72e..849a95bc 100644 --- a/tests/utils/test_Misc.py +++ b/tests/utils/test_Misc.py @@ -14,10 +14,11 @@ from osbot_utils.utils.Misc import bytes_to_base64, base64_to_bytes, date_time_now, class_name, str_to_date, get_value, \ get_random_color, is_number, none_or_empty, random_filename, random_port, random_number, random_string, \ random_string_and_numbers, str_md5, random_uuid, trim, to_int, wait, word_wrap, word_wrap_escaped, \ - convert_to_number, remove_html_tags, get_field, last_letter, random_text, random_password, split_lines, under_debugger, base64_to_str, \ + convert_to_number, remove_html_tags, get_field, last_letter, random_text, random_password, split_lines, \ + under_debugger, base64_to_str, \ str_sha256, str_to_base64, env_vars_list, env_vars, env_value, flist, ignore_warning__unclosed_ssl, list_set, \ lower, remove_multiple_spaces, split_spaces, sorted_set, upper, log_to_file, log_debug, log_error, \ - log_info, time_now, str_index, time_str_milliseconds + log_info, time_now, str_index, time_str_milliseconds, url_encode, url_decode class test_Misc(TestCase): @@ -350,6 +351,11 @@ def test_sorted_set(self): assert sorted_set({}) == [] assert sorted_set({"b", "a", "c"}) == ["a", "b", "c"] + def test_url_encode(self): + data = "https://aaa.com?aaaa=bbb&cccc=ddd+eee fff ;/n!@£$%" + assert url_encode(data) == 'https%3A%2F%2Faaa.com%3Faaaa%3Dbbb%26cccc%3Dddd%2Beee+fff+%3B%2Fn%21%40%C2%A3%24%25' + assert url_decode(url_encode(data)) == data # confirm round trip + def test_upper(self): assert upper("abc$#4ABC") == "ABC$#4ABC" assert upper("") == "" From ee18a18257e8781c3fc667ee1299826164e6c1b3 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 1 Aug 2021 13:00:24 +0100 Subject: [PATCH 05/35] using md5 for cache_as_temp key (for params) --- osbot_utils/decorators/methods/cache_on_tmp.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/osbot_utils/decorators/methods/cache_on_tmp.py b/osbot_utils/decorators/methods/cache_on_tmp.py index d19398c0..95790d87 100644 --- a/osbot_utils/decorators/methods/cache_on_tmp.py +++ b/osbot_utils/decorators/methods/cache_on_tmp.py @@ -1,3 +1,5 @@ +from osbot_utils.utils.Misc import str_md5 + from osbot_utils.utils.Files import temp_folder_current, path_combine, folder_create from osbot_utils.utils.Json import json_load_file_gz, json_save_file_gz @@ -41,7 +43,8 @@ def get_cache_in_tmp_key(self, self_obj, function_obj, params): function_name = function_obj.__name__ if params: params_as_string = '_'.join(str(x) for x in params).replace('/',' ') - return f'{class_name}_{function_name}_{params_as_string}.gz' + params_md5 = str_md5(params_as_string) + return f'{class_name}_{function_name}_{params_md5}.gz' else: return f'{class_name}_{function_name}.gz' @@ -52,9 +55,11 @@ def get_cache_in_tmp_path(self, self_obj, function_obj, params): return cache_path #return '/tmp/cache_in_tmp_{0}.gz'.format(cache_key) + # todo: refactor to use pickle for data load def get_cache_in_tmp_data(self, cache_path): return json_load_file_gz(path=cache_path) + # todo: refactor to use pickle for data save def save_cache_in_tmp_data(self, cache_path, data): json_save_file_gz(path=cache_path, python_object=data) return data \ No newline at end of file From 4fd0ab59a387e5f99b91bbee20e070e472289eab Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Mon, 2 Aug 2021 14:41:47 +0100 Subject: [PATCH 06/35] added method alias `file_create_from_bytes` --- osbot_utils/utils/Files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Files.py b/osbot_utils/utils/Files.py index 64f5f21f..5fed50d1 100644 --- a/osbot_utils/utils/Files.py +++ b/osbot_utils/utils/Files.py @@ -247,8 +247,8 @@ def write(path = None,contents=None, extension=None, mode='w'): return path @staticmethod - def write_bytes(path=None, contents=None, extension=None): - return Files.write(path=path, contents=contents, extension=extension, mode='wb') + def write_bytes(path=None, bytes=None, extension=None): + return Files.write(path=path, contents=bytes, extension=extension, mode='wb') @staticmethod def write_gz(path=None, contents=None): @@ -310,6 +310,7 @@ def zip_files(base_folder, file_pattern="*.*", target_file=None): file_delete = Files.delete file_create = Files.write file_create_bytes = Files.write_bytes +file_create_from_bytes = Files.write_bytes file_create_gz = Files.write_gz file_exists = Files.exists file_extension = Files.file_extension From 4c1c1d1c0257cdab92726a166049b26d5b43e047 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Tue, 3 Aug 2021 01:22:11 +0100 Subject: [PATCH 07/35] Added new helper class Temp_Web_Server --- osbot_utils/utils/Files.py | 122 +++++++++++++++------------ osbot_utils/utils/Temp_File.py | 8 +- osbot_utils/utils/Temp_Web_Server.py | 80 ++++++++++++++++++ tests/utils/test_Temp_File.py | 9 +- tests/utils/test_Temp_Web_Server.py | 111 ++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 55 deletions(-) create mode 100644 osbot_utils/utils/Temp_Web_Server.py create mode 100644 tests/utils/test_Temp_Web_Server.py diff --git a/osbot_utils/utils/Files.py b/osbot_utils/utils/Files.py index 5fed50d1..a21f0b04 100644 --- a/osbot_utils/utils/Files.py +++ b/osbot_utils/utils/Files.py @@ -22,6 +22,16 @@ def copy(source:str, destination:str) -> str: folder_create(parent_folder) # ensure targer folder exists return shutil.copy(source, destination) # copy file and returns file destination + @staticmethod + def contains(path, content): + text = Files.contents(path) + if type(content) is list: + for item in content: + if item not in text: + return False + return True + return content in text + @staticmethod def contents(path, mode='rt'): if file_exists(path): @@ -71,6 +81,12 @@ def files(path, pattern= '*.*'): # todo: check behaviour result.append(str(file)) # todo: see if there is a better way to do this conversion to string return sorted(result) + @staticmethod + def file_create_all_parent_folders(file_path): + parent_path = parent_folder(file_path) + path = Path(parent_path) + path.mkdir(parents=True, exist_ok=True) + return parent_path @staticmethod def file_name(path): @@ -294,58 +310,60 @@ def zip_files(base_folder, file_pattern="*.*", target_file=None): # helper methods # todo: all all methods above (including the duplicated mappings at the top) -create_folder = Files.folder_create -create_folder_in_parent = Files.folder_create_in_parent -create_temp_file = Files.write -current_folder = Files.current_folder -current_temp_folder = Files.temp_folder_current - -file_bytes = Files.bytes -file_contents = Files.contents -file_contents_gz = Files.contents_gz -file_contents_md5 = Files.contents_md5 -file_contents_sha256 = Files.contents_sha256 -file_contents_as_bytes = Files.bytes -file_copy = Files.copy -file_delete = Files.delete -file_create = Files.write -file_create_bytes = Files.write_bytes -file_create_from_bytes = Files.write_bytes -file_create_gz = Files.write_gz -file_exists = Files.exists -file_extension = Files.file_extension -file_extension_fix = Files.file_extension_fix -file_find = Files.find -file_lines = Files.lines -file_lines_gz = Files.lines_gz -file_md5 = Files.contents_md5 -file_name = Files.file_name -file_not_exists = Files.not_exists -file_open = Files.open -file_open_gz = Files.open_gz -file_open_bytes = Files.open_bytes -file_save = Files.save -file_sha256 = Files.contents_sha256 -file_size = Files.file_size -file_stats = Files.file_stats -file_write = Files.write -file_write_bytes = Files.write_bytes -file_write_gz = Files.write_gz -file_unzip = Files.unzip_file -files_list = Files.files - -folder_create = Files.folder_create -folder_create_in_parent = Files.folder_create_in_parent -folder_create_temp = Files.temp_folder -folder_copy = Files.folder_copy -folder_copy_except = Files.folder_copy -folder_delete_all = Files.folder_delete_all -folder_exists = Files.folder_exists -folder_not_exists = Files.folder_not_exists -folder_name = Files.folder_name -folder_temp = Files.temp_folder -folder_files = Files.files -folder_zip = Files.zip_folder +create_folder = Files.folder_create +create_folder_in_parent = Files.folder_create_in_parent +create_temp_file = Files.write +current_folder = Files.current_folder +current_temp_folder = Files.temp_folder_current + +file_bytes = Files.bytes +file_contains = Files.contains +file_contents = Files.contents +file_contents_gz = Files.contents_gz +file_contents_md5 = Files.contents_md5 +file_contents_sha256 = Files.contents_sha256 +file_contents_as_bytes = Files.bytes +file_create_all_parent_folders = Files.file_create_all_parent_folders +file_copy = Files.copy +file_delete = Files.delete +file_create = Files.write +file_create_bytes = Files.write_bytes +file_create_from_bytes = Files.write_bytes +file_create_gz = Files.write_gz +file_exists = Files.exists +file_extension = Files.file_extension +file_extension_fix = Files.file_extension_fix +file_find = Files.find +file_lines = Files.lines +file_lines_gz = Files.lines_gz +file_md5 = Files.contents_md5 +file_name = Files.file_name +file_not_exists = Files.not_exists +file_open = Files.open +file_open_gz = Files.open_gz +file_open_bytes = Files.open_bytes +file_save = Files.save +file_sha256 = Files.contents_sha256 +file_size = Files.file_size +file_stats = Files.file_stats +file_write = Files.write +file_write_bytes = Files.write_bytes +file_write_gz = Files.write_gz +file_unzip = Files.unzip_file +files_list = Files.files + +folder_create = Files.folder_create +folder_create_in_parent = Files.folder_create_in_parent +folder_create_temp = Files.temp_folder +folder_copy = Files.folder_copy +folder_copy_except = Files.folder_copy +folder_delete_all = Files.folder_delete_all +folder_exists = Files.folder_exists +folder_not_exists = Files.folder_not_exists +folder_name = Files.folder_name +folder_temp = Files.temp_folder +folder_files = Files.files +folder_zip = Files.zip_folder load_file = Files.contents load_file_gz = Files.contents_gz diff --git a/osbot_utils/utils/Temp_File.py b/osbot_utils/utils/Temp_File.py index 660ff8c1..85e4e3a7 100644 --- a/osbot_utils/utils/Temp_File.py +++ b/osbot_utils/utils/Temp_File.py @@ -1,4 +1,4 @@ -from osbot_utils.utils.Files import Files +from osbot_utils.utils.Files import Files, file_delete, folder_delete_all, files_list from osbot_utils.utils.Misc import random_filename @@ -14,4 +14,8 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): - Files.delete(self.file_path) + file_delete (self.file_path) + folder_delete_all(self.tmp_folder) + + def files_in_folder(self): + return files_list(self.tmp_folder) diff --git a/osbot_utils/utils/Temp_Web_Server.py b/osbot_utils/utils/Temp_Web_Server.py new file mode 100644 index 00000000..57dcabe1 --- /dev/null +++ b/osbot_utils/utils/Temp_Web_Server.py @@ -0,0 +1,80 @@ +from contextlib import contextmanager +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from urllib.parse import urljoin + +from osbot_utils.utils.Files import file_create, path_combine, temp_filename, file_create_all_parent_folders + +from osbot_utils.utils.Misc import random_port, random_string + +from osbot_utils.utils.Http import port_is_open, GET + + +class Temp_Web_Server: + server : ThreadingHTTPServer + server_thread : Thread + + def __init__(self, host: str = None, port: int = None, root_folder: str = None, server_name = None, http_handler = None): + self.host = host or "127.0.0.1" + self.port = port or random_port() + self.root_folder = root_folder or "." + self.server_name = server_name or "Temp_Web_Server" + self.http_handler = http_handler or SimpleHTTPRequestHandler + + def __enter__(self): + #params = (self.host, self.port), partial(SimpleHTTPRequestHandler, directory=self.root_folder) + if self.http_handler is SimpleHTTPRequestHandler: + handler_config = partial(self.http_handler, directory=self.root_folder) + else: + handler_config = partial(self.http_handler) + self.server = ThreadingHTTPServer((self.host, self.port), handler_config) + self.server_thread = Thread(target=self.server.serve_forever, name=self.server_name) + self.server_thread.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.server.server_close() + self.server.shutdown() + self.server_thread.join() + + def add_file(self, relative_file_path=None, file_contents=None): + if relative_file_path is None: + relative_file_path = temp_filename() + if file_contents is None: + file_contents = random_string() + full_path = path_combine(self.root_folder, relative_file_path) # todo: fix the path transversal vulnerability that exists in this function #security + file_create_all_parent_folders(full_path) + file_create(path=full_path, contents=file_contents) + return full_path + + def url(self): + return f"http://{self.host}:{self.port}" + + def server_port_open(self): + return port_is_open(host=self.host, port=self.port) + + def GET(self, path=''): + url = urljoin(self.url(), path) + try: + return GET(url) + except Exception as error: + print(error) # todo: add support for using logging + return None + + def GET_contains(self, content, path=''): + page_html = self.GET(path=path) + if type(content) is list: + for item in content: + if item not in page_html: + return False + return True + return content in page_html + + # @contextmanager + # def http_server(self, host: str, port: int, directory: str): + # + # + # try: + # yield + # finally: diff --git a/tests/utils/test_Temp_File.py b/tests/utils/test_Temp_File.py index 0c822efe..23293f61 100644 --- a/tests/utils/test_Temp_File.py +++ b/tests/utils/test_Temp_File.py @@ -1,5 +1,5 @@ from unittest import TestCase -from osbot_utils.utils.Files import Files +from osbot_utils.utils.Files import Files, file_exists, folder_exists, file_not_exists, folder_not_exists from osbot_utils.utils.Temp_File import Temp_File class test_Temp_File(TestCase): @@ -12,6 +12,13 @@ def test__init__(self): assert temp_file.tmp_folder in temp_file.file_path assert '/' == temp_file.file_path.replace(temp_file.tmp_folder,'').replace(temp_file.tmp_file,'') + def test__confirm_file_and_folder_creation_and_deletion(self): + with Temp_File() as _: + assert file_exists (_.file_path) + assert folder_exists (_.tmp_folder) + assert file_not_exists (_.file_path) + assert folder_not_exists (_.tmp_folder) + def test__using_with__no_params(self): with Temp_File() as temp: assert Files.file_extension(temp.file_path) == '.tmp' diff --git a/tests/utils/test_Temp_Web_Server.py b/tests/utils/test_Temp_Web_Server.py new file mode 100644 index 00000000..25f8fbfa --- /dev/null +++ b/tests/utils/test_Temp_Web_Server.py @@ -0,0 +1,111 @@ +from http import server +from http.server import SimpleHTTPRequestHandler +from unittest import TestCase + +from osbot_utils.utils.Files import file_contents, file_contains, file_name, file_exists, parent_folder, folder_exists, \ + file_not_exists, folder_not_exists + +from osbot_utils.utils.Misc import random_text + +from osbot_utils.utils.Temp_File import Temp_File + +from osbot_utils.utils.Dev import pprint + +from osbot_utils.utils.Http import GET, port_is_open + +from osbot_utils.utils.Temp_Web_Server import Temp_Web_Server + + +class test_Temp_Web_Server(TestCase): + + def setUp(self): + pass + + def test__enter__leave__(self): + host = "127.0.0.1" + port = 20002 + root_folder = '.' + kwargs = { "host" : host , + "port" : port , + "root_folder": root_folder } + expected_content = ['

Directory listing for /

' , + '
  • test_Temp_Web_Server.py
    • '] + temp_web_server = Temp_Web_Server(**kwargs) + with temp_web_server as web_server: + assert web_server.server_port_open() is True + assert web_server.GET_contains(expected_content) is True + assert web_server.GET_contains('\n' ) is True + assert web_server.GET_contains('aaaaaa__bbbbbb') is False + + assert web_server.server_port_open() is False + assert web_server.GET() is None + + def test_add_file(self): + with Temp_File() as temp_file: + temp_folder = temp_file.tmp_folder + with Temp_Web_Server(root_folder=temp_folder) as web_server: + assert len(temp_file.files_in_folder()) == 1 + new_file = web_server.add_file() + assert len(temp_file.files_in_folder()) == 2 + assert new_file in temp_file.files_in_folder() + + another_file_name = 'aaaa.txt' + another_contents = 'some content' + another_file_path = web_server.add_file(relative_file_path= another_file_name, file_contents=another_contents) + assert len(temp_file.files_in_folder()) == 3 + assert another_file_path in temp_file.files_in_folder() + assert file_name(another_file_path ) == another_file_name + assert file_contents(another_file_path) == another_contents + assert web_server.GET(another_file_name) == another_contents + + virtual_folder = 'some_folder/some_subfolder' + filename_in_folder = 'bbbb.txt' + relative_file_path = f'{virtual_folder}/{filename_in_folder}' + some_random_text = random_text() + file_in_virtual_folder = web_server.add_file(relative_file_path=relative_file_path, file_contents=some_random_text) + path_new_folder = parent_folder(file_in_virtual_folder) + + assert file_in_virtual_folder == f"{temp_folder}/{virtual_folder}/{filename_in_folder}" + assert file_exists(file_in_virtual_folder) + assert web_server.GET(relative_file_path) == some_random_text + + assert file_exists (new_file) + assert folder_exists(temp_folder) + assert folder_exists(path_new_folder) + assert file_not_exists(new_file) + assert folder_not_exists(temp_folder) + assert folder_not_exists(path_new_folder) + + + def test__defaults(self): + web_server = Temp_Web_Server() + assert web_server.host == "127.0.0.1" + assert type(web_server.port) is int + assert web_server.root_folder == "." + assert web_server.server_name == "Temp_Web_Server" + assert web_server.http_handler == SimpleHTTPRequestHandler + + def test__in_temp_folder(self): + temp_content = random_text() + with Temp_File(contents=temp_content) as temp_file: + with Temp_Web_Server(root_folder=temp_file.tmp_folder) as web_server: + temp_file_name = temp_file.tmp_file + assert web_server.GET_contains(temp_file_name) + assert file_contains(temp_file.file_path, temp_content) + assert web_server.GET(temp_file_name) == temp_content + + def test__with_custom_http_handler(self): + class MyHandler(server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + # Here's where all the complicated logic is done to generate HTML. + # For clarity here, replace with a simple stand-in: + html = "

    hello world

    " + + self.wfile.write(html.encode()) + + with Temp_Web_Server(http_handler=MyHandler) as web_server: + pprint(web_server.GET()) From 751d3c41713c3202c56f3d02ece312807f71c56c Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Tue, 3 Aug 2021 08:40:49 +0100 Subject: [PATCH 08/35] added Files.file_to_base64 method --- osbot_utils/utils/Files.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/osbot_utils/utils/Files.py b/osbot_utils/utils/Files.py index a21f0b04..3cb50179 100644 --- a/osbot_utils/utils/Files.py +++ b/osbot_utils/utils/Files.py @@ -8,6 +8,8 @@ from os.path import abspath, join from pathlib import Path +from osbot_utils.utils.Misc import bytes_to_base64 + class Files: @staticmethod @@ -106,6 +108,10 @@ def file_extension_fix(extension): return '.' + extension return extension + @staticmethod + def file_to_base64(path): + return bytes_to_base64(file_bytes(path)) + @staticmethod def file_size(path): return file_stats(path).st_size @@ -342,6 +348,7 @@ def zip_files(base_folder, file_pattern="*.*", target_file=None): file_open = Files.open file_open_gz = Files.open_gz file_open_bytes = Files.open_bytes +file_to_base64 = Files.file_to_base64 file_save = Files.save file_sha256 = Files.contents_sha256 file_size = Files.file_size From 8bf13af7e872daffcc5589356e7d39c4095e7a9c Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Tue, 3 Aug 2021 09:36:49 +0100 Subject: [PATCH 09/35] added Files.file_from_base64 method --- osbot_utils/utils/Files.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/osbot_utils/utils/Files.py b/osbot_utils/utils/Files.py index 3cb50179..d6163621 100644 --- a/osbot_utils/utils/Files.py +++ b/osbot_utils/utils/Files.py @@ -8,7 +8,7 @@ from os.path import abspath, join from pathlib import Path -from osbot_utils.utils.Misc import bytes_to_base64 +from osbot_utils.utils.Misc import bytes_to_base64, base64_to_bytes class Files: @@ -112,6 +112,11 @@ def file_extension_fix(extension): def file_to_base64(path): return bytes_to_base64(file_bytes(path)) + @staticmethod + def file_from_base64(bytes_base64, path=None, extension=None): + bytes_ = base64_to_bytes(bytes_base64) + return file_create_bytes(bytes=bytes_, path=path, extension=None) + @staticmethod def file_size(path): return file_stats(path).st_size @@ -349,6 +354,7 @@ def zip_files(base_folder, file_pattern="*.*", target_file=None): file_open_gz = Files.open_gz file_open_bytes = Files.open_bytes file_to_base64 = Files.file_to_base64 +file_from_base64 = Files.file_from_base64 file_save = Files.save file_sha256 = Files.contents_sha256 file_size = Files.file_size From c61e5619ae3767e4e1aeb494690902509b269ef4 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 8 Aug 2021 20:47:06 +0100 Subject: [PATCH 10/35] made Fluent_Dict keys value to be sorted (to help with unit tests) --- osbot_utils/fluent/Fluent_Dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osbot_utils/fluent/Fluent_Dict.py b/osbot_utils/fluent/Fluent_Dict.py index 2a64e6e7..4348a2fe 100644 --- a/osbot_utils/fluent/Fluent_Dict.py +++ b/osbot_utils/fluent/Fluent_Dict.py @@ -10,7 +10,7 @@ def __init__(self,*args,**kwargs): super().__init__(*args,**kwargs) def keys(self): - return Fluent_List(list(self.data.keys())) + return Fluent_List(sorted(list(self.data.keys()))) def size(self): return len(self.data) From 1912ff71a144cc667f0313077e3af8fd6340c7fd Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 15 Aug 2021 16:18:49 +0100 Subject: [PATCH 11/35] add couple helper methods for getting an object's data --- osbot_utils/utils/Misc.py | 29 ++++++++++++++++++++++ tests/utils/test_Misc.py | 51 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index f4b77239..13ffd619 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -153,10 +153,14 @@ def is_number(value): def ignore_warning__unclosed_ssl(): warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + def last_letter(text): if text and (type(text) is str) and len(text) > 0: return text[-1] +def len_list(target): + return len(list(target)) + def list_add(array : list, value): if value is not None: array.append(value) @@ -208,6 +212,9 @@ def list_pop_and_trim(array, position=None): def list_set(target): return sorted(list(set(target))) +def list_set_dict(target): + return sorted(list(set(obj_dict(target)))) + def list_filter(target_list, filter_function): return list(filter(filter_function, target_list)) @@ -262,6 +269,28 @@ def lower(target : str): return target.lower() return "" +def obj_dict(target=None): + if target and hasattr(target,'__dict__'): + return target.__dict__ + return {} + +def obj_items(target=None): + return list(obj_dict(target).items()) + +def obj_keys(target=None): + return list(obj_dict(target).keys()) + +def obj_get_value(target=None, key=None, default=None): + return get_field(target=target, field=key, default=default) + +def obj_values(target=None): + return list(obj_dict(target).values()) + + +def size(target=None): + if target and hasattr(target, '__len__'): + return len(target) + return 0 def str_index(target:str, source:str): try: diff --git a/tests/utils/test_Misc.py b/tests/utils/test_Misc.py index 849a95bc..7e280c24 100644 --- a/tests/utils/test_Misc.py +++ b/tests/utils/test_Misc.py @@ -18,7 +18,8 @@ under_debugger, base64_to_str, \ str_sha256, str_to_base64, env_vars_list, env_vars, env_value, flist, ignore_warning__unclosed_ssl, list_set, \ lower, remove_multiple_spaces, split_spaces, sorted_set, upper, log_to_file, log_debug, log_error, \ - log_info, time_now, str_index, time_str_milliseconds, url_encode, url_decode + log_info, time_now, str_index, time_str_milliseconds, url_encode, url_decode, obj_dict, obj_items, obj_keys, \ + obj_values, obj_get_value, size class test_Misc(TestCase): @@ -140,7 +141,35 @@ def test_logger_add_handler__file(self): log_info ('info') assert file_contents(log_file) == 'error\ninfo\n' - + def test_obj_dict(self): + class Target: + def __init__(self): + self.var_1 = 'the answer' + self.var_2 = 'is' + self.var_3 = 42 + assert obj_dict (Target()) == {'var_1': 'the answer', 'var_2': 'is', 'var_3': 42} + assert obj_items (Target()) == [('var_1', 'the answer'), ('var_2', 'is'), ('var_3', 42)] + assert obj_keys (Target()) == ['var_1', 'var_2', 'var_3'] + assert obj_values(Target()) == ['the answer', 'is', 42] + target = Target() + for key,value in obj_items(target): + assert obj_get_value(target, key ) == value + assert obj_get_value(target, key , 'aa') == value + assert obj_get_value(target, key+'a', 'aa') == 'aa' + + # check cases when bad data is submitted + assert obj_dict () == {} + assert obj_items () == [] + assert obj_keys () == [] + assert obj_values() == [] + assert obj_dict (42) == {} + assert obj_items (42) == [] + assert obj_keys (42) == [] + assert obj_values({}) == [] + assert obj_dict ({}) == {} + assert obj_items ({}) == [] + assert obj_keys ({}) == [] + assert obj_values({}) == [] def test_none_or_empty(self): assert none_or_empty(None, None) is True @@ -150,6 +179,24 @@ def test_none_or_empty(self): assert none_or_empty({'a': 42}, 'b') is True assert none_or_empty({'a': 42}, 'a') is False + def test_size(self): + assert size( ) == 0 + assert size(0 ) == 0 + assert size('' ) == 0 + assert size(None) == 0 + assert size('1' ) == 1 + assert size(1 ) == 0 + assert size('2' ) == 1 + assert size(2 ) == 0 + assert size('22') == 2 + + assert size([] ) == 0 + assert size([0] ) == 1 + assert size([0,1]) == 2 + + assert size({} ) == 0 + assert size({'a':0}) == 1 + def test_random_filename(self): result = random_filename() assert len(result) == 14 From f7fd777c7393d01369477ca780c77f226587a66f Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 15 Aug 2021 19:37:40 +0100 Subject: [PATCH 12/35] Fixed broken tests --- tests/decorators/methods/test_cache_on_tmp.py | 8 ++++++-- tests/utils/test_Files.py | 2 +- tests/utils/test_Temp_Web_Server.py | 7 +++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/decorators/methods/test_cache_on_tmp.py b/tests/decorators/methods/test_cache_on_tmp.py index abc2806b..e6e42136 100644 --- a/tests/decorators/methods/test_cache_on_tmp.py +++ b/tests/decorators/methods/test_cache_on_tmp.py @@ -1,6 +1,8 @@ from pprint import pprint from unittest import TestCase +from osbot_utils.utils.Misc import str_md5 + from osbot_utils.utils.Files import file_name from osbot_utils.utils.Json import json_load_file_gz @@ -49,9 +51,11 @@ def test_cache_on_tmp(self): assert 'osbot_cache_on_tmp/An_Class_an_function.gz' in an_class.an_function() def test_cache_on_tmp__with_params(self): + param = 'aaaaa' with Profiler() as profiler: - assert An_Class().an_function_with_params('aaaaa') == 'aaaaa' + assert An_Class().an_function_with_params(param) == param cache_on_tmp_self = profiler.get_f_locals_variable('self') - assert file_name(cache_on_tmp_self.last_cache_path) == 'An_Class_an_function_with_params_aaaaa.gz' \ No newline at end of file + temp_file_name= f'An_Class_an_function_with_params_{str_md5(param)}.gz' + assert file_name(cache_on_tmp_self.last_cache_path) == temp_file_name \ No newline at end of file diff --git a/tests/utils/test_Files.py b/tests/utils/test_Files.py index 8b998766..c1a57c37 100644 --- a/tests/utils/test_Files.py +++ b/tests/utils/test_Files.py @@ -119,7 +119,7 @@ def test_file_write(self): def test_file_write_bytes(self): target = temp_file() bytes = b"\x89PNG___" - assert file_bytes(file_write_bytes(target, contents=bytes)) == bytes + assert file_bytes(file_write_bytes(target, bytes=bytes)) == bytes assert file_open_bytes(target).read() == b'\x89PNG___' diff --git a/tests/utils/test_Temp_Web_Server.py b/tests/utils/test_Temp_Web_Server.py index 25f8fbfa..f3a88809 100644 --- a/tests/utils/test_Temp_Web_Server.py +++ b/tests/utils/test_Temp_Web_Server.py @@ -3,7 +3,7 @@ from unittest import TestCase from osbot_utils.utils.Files import file_contents, file_contains, file_name, file_exists, parent_folder, folder_exists, \ - file_not_exists, folder_not_exists + file_not_exists, folder_not_exists, current_temp_folder from osbot_utils.utils.Misc import random_text @@ -24,12 +24,11 @@ def setUp(self): def test__enter__leave__(self): host = "127.0.0.1" port = 20002 - root_folder = '.' + root_folder = current_temp_folder() kwargs = { "host" : host , "port" : port , "root_folder": root_folder } - expected_content = ['

    Directory listing for /

    ' , - '
  • test_Temp_Web_Server.py
    • '] + expected_content = ['

    Directory listing for /

    '] temp_web_server = Temp_Web_Server(**kwargs) with temp_web_server as web_server: assert web_server.server_port_open() is True From 21b5f85525d3b886a39e68af147b9888aaf3b282 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 15 Aug 2021 19:44:54 +0100 Subject: [PATCH 13/35] Added util class Python_Logger --- osbot_utils/utils/Python_Logger.py | 177 +++++++++++++++++++++ tests/utils/test_Python_Logger.py | 239 +++++++++++++++++++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 osbot_utils/utils/Python_Logger.py create mode 100644 tests/utils/test_Python_Logger.py diff --git a/osbot_utils/utils/Python_Logger.py b/osbot_utils/utils/Python_Logger.py new file mode 100644 index 00000000..9fdf1bb1 --- /dev/null +++ b/osbot_utils/utils/Python_Logger.py @@ -0,0 +1,177 @@ +import logging +import sys +from logging import Logger +from logging.handlers import MemoryHandler + +from osbot_utils.decorators.lists.group_by import group_by +from osbot_utils.decorators.lists.index_by import index_by +from osbot_utils.utils.Misc import random_string, obj_dict +from osbot_utils.utils.Files import temp_file + +DEFAULT_LOG_LEVEL = logging.DEBUG +DEFAULT_LOG_FORMAT = '%(asctime)s\t|\t%(name)s\t|\t%(levelname)s\t|\t%(message)s' +MEMORY_LOGGER_CAPACITY = 1024*10 +MEMORY_LOGGER_FLUSH_LEVEL = logging.ERROR + +class Python_Logger_Config: + def __init__(self): + self.elastic_host = None + self.elastic_password = None + self.elastic_port = None + self.elastic_username = None + #self.log_to_aws_s3 = False # todo + #self.log_to_aws_cloud_trail = False # todo + #self.log_to_aws_firehose = False # todo + self.log_to_console = False # todo + self.log_to_file = False # todo + #self.log_to_elastic = False # todo + self.log_to_memory = False + self.path_logs = None + self.log_format = DEFAULT_LOG_FORMAT + self.log_level = DEFAULT_LOG_LEVEL + + +class Python_Logger: + config : Python_Logger_Config + logger : Logger + def __init__(self, logger_name= None, logger_config : Python_Logger_Config = None): + self.logger_name = logger_name or random_string(prefix="Python_Logger_") + self.set_config(logger_config) + + def manager_get_loggers(self): + return Logger.manager.loggerDict + + def manager_remove_logger(self): + logger_dict = Logger.manager.loggerDict + if self.logger_name in logger_dict: # need to do it manually here since Logger.manager doesn't seem to have a way to remove loggers + del logger_dict[self.logger_name] + return True + return False + + def setup(self): + self.logger = logging.getLogger(self.logger_name) + self.set_log_level() + self.add_handler_memory() + return self + + # Setters + def set_config(self, config): + if type(config) is Python_Logger_Config: + self.config = config + else: + self.config = Python_Logger_Config() + return self.config + + def set_log_level(self, level=None): + level = level or self.config.log_level + if self.logger: + self.logger.setLevel(level) + return True + return False + + # Getters + def log_handlers(self): + if self.logger: + return self.logger.handlers + return [] + + def log_handler(self, handler_type): + for handler in self.log_handlers(): + if type(handler) is handler_type: + return handler + return None + + def log_handler_console(self): + return self.log_handler(logging.StreamHandler) + + def log_handler_file(self): + return self.log_handler(logging.FileHandler) + + def log_handler_memory(self): + return self.log_handler(MemoryHandler) + + def log_formatter(self): + return logging.Formatter(self.config.log_format) + + def log_level(self): + return self.config.log_level + + # Actions + + def add_console_logger(self): + self.config.log_to_console = True + return self.add_handler_console() + + def add_memory_logger(self): + self.config.log_to_memory = True + return self.add_handler_memory() + + def add_file_logger(self,path_log_file=None): + self.config.log_to_file = True + return self.add_handler_file(path_log_file=path_log_file) + + # Handlers + def add_handler_console(self): + if self.logger and self.config.log_to_console: + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + handler.setFormatter(self.log_formatter()) + self.logger.addHandler(handler) + return True + return False + + def add_handler_file(self, path_log_file=None): + if self.logger and self.config.log_to_file: + if path_log_file is None: + path_log_file = temp_file(extension='.log') + handler = logging.FileHandler(path_log_file) + handler.setLevel(self.log_level()) + handler.setFormatter(self.log_formatter()) + self.logger.addHandler(handler) + return True + return False + + def add_handler_memory(self): + if self.logger and self.config.log_to_memory: + capacity = MEMORY_LOGGER_CAPACITY + flush_level = MEMORY_LOGGER_FLUSH_LEVEL + target = None # we want the messages to only be kept in memory + memory_handler = MemoryHandler(capacity=capacity, flushLevel=flush_level, target=target,flushOnClose=True) + memory_handler.setLevel(self.log_level()) + self.logger.addHandler(memory_handler) + return True + return False + + # Utils + def memory_handler_exceptions(self): + return self.memory_handler_logs(index_by='levelname').get('EXCEPTIONS', {}) + + @index_by + @group_by + def memory_handler_logs(self): + logs = [] + memory_handler = self.log_handler_memory() + if memory_handler: + for log_record in memory_handler.buffer: + logs.append(obj_dict(log_record)) + return logs + + def memory_handler_messages(self): + return [log_entry.get('message') for log_entry in self.memory_handler_logs()] + + + # Logging methods + + def debug (self, msg='', *args, **kwargs): return self.log('debug' , msg, *args, **kwargs) + def info (self, msg='', *args, **kwargs): return self.log('info' , msg, *args, **kwargs) + def warning (self, msg='', *args, **kwargs): return self.log('warning' , msg, *args, **kwargs) + def error (self, msg='', *args, **kwargs): return self.log('error' , msg, *args, **kwargs) + def exception(self, msg='', *args, **kwargs): return self.log('exception' , msg, *args, **kwargs) + def critical (self, msg='', *args, **kwargs): return self.log('critical' , msg, *args, **kwargs) + + def log(self, level, msg, *args, **kwargs): + if self.logger: + log_method = getattr(self.logger, level) + log_method(msg, *args, **kwargs) + return True + return False diff --git a/tests/utils/test_Python_Logger.py b/tests/utils/test_Python_Logger.py new file mode 100644 index 00000000..e5c355c7 --- /dev/null +++ b/tests/utils/test_Python_Logger.py @@ -0,0 +1,239 @@ +from _thread import RLock +from datetime import datetime +from io import TextIOWrapper, StringIO +from logging import Logger, LogRecord +from logging.handlers import MemoryHandler +from unittest import TestCase +from unittest.mock import patch + +import pytest + +from osbot_utils.utils.Json import json_load_file +from osbot_utils.utils.Files import file_lines, file_exists, file_contents, file_delete +from osbot_utils.utils.Python_Logger import Python_Logger, Python_Logger_Config, DEFAULT_LOG_LEVEL, \ + MEMORY_LOGGER_CAPACITY, MEMORY_LOGGER_FLUSH_LEVEL +from osbot_utils.utils.Misc import len_list, list_set, obj_dict, obj_items, size, list_set_dict + + +class test_Python_Logger_Config(TestCase): + def setUp(self): + self.config = Python_Logger_Config() + + def test__init__(self): + assert obj_dict(self.config) == {'elastic_host' : None , + 'elastic_password' : None , + 'elastic_port' : None , + 'elastic_username' : None , + 'log_level' : 10 , + 'log_format' : '%(asctime)s\t|\t%(name)s\t|\t%(levelname)s\t|\t%(message)s', + 'log_to_console' : False , + 'log_to_file' : False , + 'log_to_memory' : False , + 'path_logs' : None } + + +class test_Python_Logger(TestCase): + + def setUp(self): + self.logger = Python_Logger().setup() + + def tearDown(self) -> None: + self.logger.manager_remove_logger() + + def test__init__(self): + assert obj_items(self.logger.config) == obj_items(Python_Logger_Config()) + assert self.logger.logger is not None + + # Helpers and Setup methods + + def test_manager_get_loggers(self): + loggers = list_set(self.logger.manager_get_loggers()) + assert len(loggers) > 5 + assert self.logger.logger_name in loggers + assert 'dotenv' in loggers + assert 'dotenv.main' in loggers + + def test_setup(self): # setup is called as part of the Unit Tests setUp() with default values + config = self.logger.config + logger = self.logger.logger + + assert type(logger) == Logger + assert logger.disabled == False + assert logger.filters == [] + assert logger.level == config.log_level + assert logger.propagate == True + assert logger.handlers == [] + assert logger.name.startswith("Python_Logger") is True + + @pytest.mark.skip + def test_get_logger(self): + logger = self.logger.logger() + + logger.info('this is an info message') + + assert len_list(file_lines(self.logger.logger_file)) == 1 + log_data = json_load_file(self.logger.logger_file) + assert list_set(log_data) == ['@timestamp', 'ecs', 'log', 'log.level', 'message', 'process'] + assert log_data.get('message') == 'this is an info message' + + # Logging methods + + + + @patch('sys.stdout', new_callable=StringIO) + def test_add_console_logger(self, sys_stdout): + assert self.logger.add_console_logger() is True + console_handler = self.logger.log_handler_console() + obj_data = obj_dict(console_handler) + assert obj_data == { '_name' : None , + 'filters' : [] , + 'formatter' : obj_data.get('formatter') , + 'level' : DEFAULT_LOG_LEVEL , + 'lock' : obj_data.get('lock') , + 'stream' : obj_data.get('stream') } + + debug_message = 'an debug message' + self.logger.debug(debug_message) + log_entry = sys_stdout.getvalue().split('\t|\t') + assert len(log_entry) == 4 + assert str(datetime.now().minute) in log_entry[0] + assert self.logger.logger_name == log_entry[1] + assert 'DEBUG' == log_entry[2] + assert f'{debug_message}\n' == log_entry[3] + + def test_add_file_logger(self): + assert self.logger.add_file_logger() is True + file_handler = self.logger.log_handler_file() + log_file = file_handler.baseFilename + obj_data = obj_dict(file_handler) + assert obj_data== {'_name' : None , + 'baseFilename' : obj_data.get('baseFilename'), + 'delay' : False , + 'encoding' : None , + 'filters' : [] , + 'formatter' : obj_data.get('formatter') , + 'level' : DEFAULT_LOG_LEVEL , + 'lock' : obj_data.get('lock') , + 'mode' : 'a' , + 'stream' : obj_data.get('stream') } + + assert file_exists(log_file) is True + assert file_contents(log_file) == '' + assert type(obj_data.get('stream')) == TextIOWrapper + + + info_message = 'an info message' + self.logger.info(info_message) + + log_message_items = file_contents(log_file).split('\t|\t') + assert len(log_message_items) == 4 + assert datetime.now().strftime('%Y-%m-%d %H:%M:%S') in log_message_items[0] + assert self.logger.logger_name == log_message_items[1] + assert 'INFO' == log_message_items[2] + assert f'{info_message}\n' == log_message_items[3] + + self.logger.info('an debug message') + log_lines = list(file_lines(log_file)) + assert len(log_lines) == 2 + assert 'an debug message\n' in log_lines.pop() + assert file_delete(log_file) + + + def test_add_memory_logger(self): + mem_handler : MemoryHandler + assert self.logger.config.log_to_memory is False + assert self.logger.add_handler_memory() is False + assert self.logger.add_memory_logger () is True + + mem_handler = self.logger.log_handler(MemoryHandler) + assert size(self.logger.log_handlers()) == 1 + assert mem_handler is not None + obj_data = obj_dict(mem_handler) + assert type(obj_data['lock']) is RLock + + assert obj_data == { '_name' : None , + 'buffer' : [] , + 'capacity' : MEMORY_LOGGER_CAPACITY , + 'filters' : [] , + 'flushLevel' : MEMORY_LOGGER_FLUSH_LEVEL , + 'flushOnClose': True , + 'formatter' : None , + 'level' : DEFAULT_LOG_LEVEL , + 'lock' : obj_data['lock' ] , + 'target' : None } + + assert mem_handler.buffer == [] + self.logger.info('an info message') + assert size(mem_handler.buffer ) == 1 + log_record = mem_handler.buffer.pop() + assert type(log_record) is LogRecord + + assert list_set_dict(log_record) == [ 'args', 'created', 'exc_info', 'exc_text', 'filename', + 'funcName', 'levelname', 'levelno', 'lineno', 'message', + 'module', 'msecs', 'msg', 'name', 'pathname', 'process', + 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName'] + + assert log_record.message == 'an info message' + + def test_memory_handler_messages(self): + import requests + urllib3_logger = Python_Logger(logger_name='urllib3').setup() # get urllib3 logger + urllib3_logger.add_memory_logger() # add a memory logger + requests.get('https://www.google.com/aaa') # make a request to Google which uses urllib3 + assert urllib3_logger.memory_handler_messages() == ['Starting new HTTPS connection (1): www.google.com:443', + 'https://www.google.com:443 "GET /aaa HTTP/1.1" 404 1564'] + + def test_debug__info__warning__error__critical(self): + self.logger.add_memory_logger() + assert self.logger.debug ('debug message' ) is True + assert self.logger.info ('info message' ) is True + assert self.logger.warning ('warning message' ) is True + assert self.logger.error ('error message' ) is True + assert self.logger.critical('critical message' ) is True + assert self.logger.memory_handler_messages() == [ 'debug message' , + 'info message' , + 'warning message' , + 'error message' , + 'critical message'] + log_messages = self.logger.memory_handler_logs(group_by='levelname') + + assert list_set(log_messages) == ['CRITICAL', 'DEBUG', 'ERROR', 'INFO', 'WARNING'] + critical_logs = log_messages.get('CRITICAL') + debug_logs = log_messages.get('DEBUG') + error_logs = log_messages.get('ERROR') + info_logs = log_messages.get('INFO') + warning_logs = log_messages.get('WARNING') + + def assert_log(log_entries, levelname, levelno, message): + assert size(log_entries) == 1 + log_entry = log_entries.pop() + assert log_entry.get('levelname') == levelname + assert log_entry.get('levelno' ) == levelno + assert log_entry.get('message' ) == message + assert list_set(log_entry) == [ 'args', 'created', 'exc_info', 'exc_text', 'filename', 'funcName', 'levelname', 'levelno', 'lineno', 'message', 'module', 'msecs', 'msg', 'name', 'pathname', 'process', 'processName', 'relativeCreated', 'stack_info', 'thread', 'threadName' ] + + assert_log(critical_logs, 'CRITICAL', 50, 'critical message') + assert_log(debug_logs , 'DEBUG' , 10, 'debug message' ) + assert_log(error_logs , 'ERROR' , 40, 'error message' ) + assert_log(info_logs , 'INFO' , 20, 'info message' ) + assert_log(warning_logs , 'WARNING' , 30, 'warning message' ) + + def test_exception(self): + self.logger.add_memory_logger() + assert self.logger.memory_handler_exceptions() == {} + self.logger.exception() # log with no params outside an exception + log_exception = self.logger.memory_handler_logs().pop() + assert log_exception.get('exc_info' ) == (None, None, None) + assert log_exception.get('exc_text' ) == 'NoneType: None' + assert log_exception.get('levelname') =='ERROR' + assert log_exception.get('message' ) == '' + try: + 40 / 0 + except Exception as error: + self.logger.logger.exception(error) + + log_exception = self.logger.memory_handler_logs().pop() + + assert str(log_exception.get('exc_info')[0]) == "" + assert 'ZeroDivisionError: division by zero' in log_exception.get('exc_text') + assert log_exception.get('message') == 'division by zero' \ No newline at end of file From 663dc3049ad9e90185d55d369244449018fc9869 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 15 Aug 2021 19:45:18 +0100 Subject: [PATCH 14/35] removed code not needed --- tests/utils/test_Python_Logger.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/utils/test_Python_Logger.py b/tests/utils/test_Python_Logger.py index e5c355c7..d4361b6d 100644 --- a/tests/utils/test_Python_Logger.py +++ b/tests/utils/test_Python_Logger.py @@ -65,17 +65,6 @@ def test_setup(self): # setup is called as part of the assert logger.handlers == [] assert logger.name.startswith("Python_Logger") is True - @pytest.mark.skip - def test_get_logger(self): - logger = self.logger.logger() - - logger.info('this is an info message') - - assert len_list(file_lines(self.logger.logger_file)) == 1 - log_data = json_load_file(self.logger.logger_file) - assert list_set(log_data) == ['@timestamp', 'ecs', 'log', 'log.level', 'message', 'process'] - assert log_data.get('message') == 'this is an info message' - # Logging methods From fe7f1bd837d56f79c66a49a216ec5d5a88196a05 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 15 Aug 2021 20:15:26 +0100 Subject: [PATCH 15/35] added set_log_format method to Python_Logger --- osbot_utils/utils/Python_Logger.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osbot_utils/utils/Python_Logger.py b/osbot_utils/utils/Python_Logger.py index 9fdf1bb1..4c05af89 100644 --- a/osbot_utils/utils/Python_Logger.py +++ b/osbot_utils/utils/Python_Logger.py @@ -62,6 +62,10 @@ def set_config(self, config): self.config = Python_Logger_Config() return self.config + def set_log_format(self, format): + if format: + self.config.log_format = format + def set_log_level(self, level=None): level = level or self.config.log_level if self.logger: From 6de7653c5e519aa7027404d167b7d3150c68d1d6 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 15 Aug 2021 22:58:13 +0100 Subject: [PATCH 16/35] minor class import refactoring --- osbot_utils/utils/Python_Logger.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/osbot_utils/utils/Python_Logger.py b/osbot_utils/utils/Python_Logger.py index 4c05af89..c8fc93c0 100644 --- a/osbot_utils/utils/Python_Logger.py +++ b/osbot_utils/utils/Python_Logger.py @@ -1,6 +1,6 @@ import logging import sys -from logging import Logger +from logging import Logger, StreamHandler, FileHandler from logging.handlers import MemoryHandler from osbot_utils.decorators.lists.group_by import group_by @@ -86,7 +86,7 @@ def log_handler(self, handler_type): return None def log_handler_console(self): - return self.log_handler(logging.StreamHandler) + return self.log_handler(StreamHandler) def log_handler_file(self): return self.log_handler(logging.FileHandler) @@ -117,7 +117,7 @@ def add_file_logger(self,path_log_file=None): # Handlers def add_handler_console(self): if self.logger and self.config.log_to_console: - handler = logging.StreamHandler(sys.stdout) + handler = StreamHandler(sys.stdout) handler.setLevel(logging.DEBUG) handler.setFormatter(self.log_formatter()) self.logger.addHandler(handler) @@ -128,7 +128,7 @@ def add_handler_file(self, path_log_file=None): if self.logger and self.config.log_to_file: if path_log_file is None: path_log_file = temp_file(extension='.log') - handler = logging.FileHandler(path_log_file) + handler = FileHandler(path_log_file) handler.setLevel(self.log_level()) handler.setFormatter(self.log_formatter()) self.logger.addHandler(handler) @@ -163,7 +163,6 @@ def memory_handler_logs(self): def memory_handler_messages(self): return [log_entry.get('message') for log_entry in self.memory_handler_logs()] - # Logging methods def debug (self, msg='', *args, **kwargs): return self.log('debug' , msg, *args, **kwargs) From 7e216e68c6e1b9f8465de2c2973457d1ff49bcbf Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Mon, 16 Aug 2021 03:00:44 +0100 Subject: [PATCH 17/35] improved Python_Logger added wait_for_port and wait_for_ssh methods --- osbot_utils/utils/Http.py | 24 ++++++++++++++--- osbot_utils/utils/Python_Logger.py | 43 +++++++++++++++++++++--------- tests/utils/test_Http.py | 8 +++++- tests/utils/test_Python_Logger.py | 19 ++++++++----- 4 files changed, 70 insertions(+), 24 deletions(-) diff --git a/osbot_utils/utils/Http.py b/osbot_utils/utils/Http.py index e46bc7cb..caeaab50 100644 --- a/osbot_utils/utils/Http.py +++ b/osbot_utils/utils/Http.py @@ -4,7 +4,9 @@ from urllib.request import Request, urlopen from osbot_utils.utils.Files import save_bytes_as_file, file_size, file_bytes, file_open_bytes +from osbot_utils.utils.Python_Logger import Python_Logger +logger = Python_Logger('OSBot-utils').setup() def current_host_online(url_to_use='http://www.google.com'): try: @@ -20,10 +22,15 @@ def is_port_open(host, port, timeout=0.5): return port_is_open(host=host, port=port, timeout=timeout) def port_is_open(port : int , host='0.0.0.0', timeout=1.0): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(timeout) - result = sock.connect_ex((host, port)) - return result == 0 + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + result = sock.connect_ex((host, port)) + return result == 0 + except: + logger.exception(f'port {port} was closed in server {host}') + return False + def Http_Request(url, data=None, headers=None, method='GET', encoding = 'utf-8', return_response_object=False): headers = headers or {} @@ -50,6 +57,15 @@ def Http_Request(url, data=None, headers=None, method='GET', encoding = 'utf-8', def port_is_not_open(port, host='0.0.0.0', timeout=1.0): return port_is_open(port, host,timeout) is False +def wait_for_ssh(host, max_attempts=20, wait_for=0.1): + return wait_for_port(host=host, port=22, max_attempts=20, wait_for=0.1) + +def wait_for_port(host, port, max_attempts=20, wait_for=0.1): + for i in range(max_attempts): + if is_port_open(host=host,port=port,timeout=wait_for): + return True + return False + def DELETE(url, data=None, headers=None): return Http_Request(url, data, headers, 'DELETE') diff --git a/osbot_utils/utils/Python_Logger.py b/osbot_utils/utils/Python_Logger.py index c8fc93c0..3e26197a 100644 --- a/osbot_utils/utils/Python_Logger.py +++ b/osbot_utils/utils/Python_Logger.py @@ -50,10 +50,27 @@ def manager_remove_logger(self): def setup(self): self.logger = logging.getLogger(self.logger_name) + self.setup_log_methods(self) self.set_log_level() self.add_handler_memory() return self + def setup_log_methods(self, target): + # adds these helper methods like this so that the filename and function values are accurate + setattr(target, "debug" , self.logger.debug ) + setattr(target, "info" , self.logger.info ) + setattr(target, "warning" , self.logger.warning ) + setattr(target, "error" , self.logger.error ) + setattr(target, "exception" , self.logger.exception ) + setattr(target, "critical" , self.logger.critical ) + + # self.info = self.logger.info + # self.warning = self.logger.warning + # self.error = self.logger.error + # self.exception = self.logger.exception + # self.critical = self.logger.critical + + # Setters def set_config(self, config): if type(config) is Python_Logger_Config: @@ -165,16 +182,16 @@ def memory_handler_messages(self): # Logging methods - def debug (self, msg='', *args, **kwargs): return self.log('debug' , msg, *args, **kwargs) - def info (self, msg='', *args, **kwargs): return self.log('info' , msg, *args, **kwargs) - def warning (self, msg='', *args, **kwargs): return self.log('warning' , msg, *args, **kwargs) - def error (self, msg='', *args, **kwargs): return self.log('error' , msg, *args, **kwargs) - def exception(self, msg='', *args, **kwargs): return self.log('exception' , msg, *args, **kwargs) - def critical (self, msg='', *args, **kwargs): return self.log('critical' , msg, *args, **kwargs) - - def log(self, level, msg, *args, **kwargs): - if self.logger: - log_method = getattr(self.logger, level) - log_method(msg, *args, **kwargs) - return True - return False + # def debug (self, msg='', *args, **kwargs): return self._log('debug' , msg, *args, **kwargs) + # #def info (self, msg='', *args, **kwargs): return self.__log__('info' , msg, *args, **kwargs) + # def warning (self, msg='', *args, **kwargs): return self._log('warning' , msg, *args, **kwargs) + # def error (self, msg='', *args, **kwargs): return self._log('error' , msg, *args, **kwargs) + # def exception(self, msg='', *args, **kwargs): return self._log('exception' , msg, *args, **kwargs) + # def critical (self, msg='', *args, **kwargs): return self._log('critical' , msg, *args, **kwargs) + # + # def __log__(self, level, msg, *args, **kwargs): + # if self.logger: + # log_method = getattr(self.logger, level) + # log_method(msg, *args, **kwargs) + # return True + # return False diff --git a/tests/utils/test_Http.py b/tests/utils/test_Http.py index a67cb68e..3d9f5d73 100644 --- a/tests/utils/test_Http.py +++ b/tests/utils/test_Http.py @@ -1,10 +1,12 @@ from pprint import pprint from unittest import TestCase +from osbot_utils.utils import Http + from osbot_utils.utils.Files import temp_file, file_not_exists, file_exists, file_bytes, file_size, file_create_bytes from osbot_utils.utils.Http import DELETE, POST, GET, GET_json, DELETE_json, GET_bytes, GET_bytes_to_file, \ dns_ip_address, port_is_open, port_is_not_open, current_host_online, POST_json, OPTIONS, PUT_json, \ - is_port_open + is_port_open, wait_for_port # using httpbin.org because it seems to be the best option @@ -31,6 +33,10 @@ def test_current_host_online(self): assert current_host_online() is True assert current_host_online(url_to_use='http://111-2222-3333-abc.com') is False + def test_wait_for_port(self): + assert wait_for_port('www.google.com', 443 ) is True + assert wait_for_port('bad-ip' , 443, max_attempts=2) is False + def test_DELETE_json(self): url = self.url_template.format(method="delete") response = DELETE_json(url, headers=self.headers, data=self.data) diff --git a/tests/utils/test_Python_Logger.py b/tests/utils/test_Python_Logger.py index d4361b6d..ce5e69fe 100644 --- a/tests/utils/test_Python_Logger.py +++ b/tests/utils/test_Python_Logger.py @@ -174,11 +174,18 @@ def test_memory_handler_messages(self): def test_debug__info__warning__error__critical(self): self.logger.add_memory_logger() - assert self.logger.debug ('debug message' ) is True - assert self.logger.info ('info message' ) is True - assert self.logger.warning ('warning message' ) is True - assert self.logger.error ('error message' ) is True - assert self.logger.critical('critical message' ) is True + assert self.logger.debug == self.logger.logger.debug + assert self.logger.info == self.logger.logger.info + assert self.logger.warning == self.logger.logger.warning + assert self.logger.error == self.logger.logger.error + assert self.logger.critical == self.logger.logger.critical + assert self.logger.exception == self.logger.logger.exception + + assert self.logger.debug ('debug message' ) is None + assert self.logger.info ('info message' ) is None + assert self.logger.warning ('warning message' ) is None + assert self.logger.error ('error message' ) is None + assert self.logger.critical('critical message' ) is None assert self.logger.memory_handler_messages() == [ 'debug message' , 'info message' , 'warning message' , @@ -210,7 +217,7 @@ def assert_log(log_entries, levelname, levelno, message): def test_exception(self): self.logger.add_memory_logger() assert self.logger.memory_handler_exceptions() == {} - self.logger.exception() # log with no params outside an exception + self.logger.exception('') # log with no params outside an exception log_exception = self.logger.memory_handler_logs().pop() assert log_exception.get('exc_info' ) == (None, None, None) assert log_exception.get('exc_text' ) == 'NoneType: None' From 20bb12a632f9c1ce55b36100e6280aeb9158f646 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Mon, 16 Aug 2021 06:50:21 +0100 Subject: [PATCH 18/35] fixed bug --- osbot_utils/utils/Http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Http.py b/osbot_utils/utils/Http.py index caeaab50..470a3139 100644 --- a/osbot_utils/utils/Http.py +++ b/osbot_utils/utils/Http.py @@ -57,8 +57,8 @@ def Http_Request(url, data=None, headers=None, method='GET', encoding = 'utf-8', def port_is_not_open(port, host='0.0.0.0', timeout=1.0): return port_is_open(port, host,timeout) is False -def wait_for_ssh(host, max_attempts=20, wait_for=0.1): - return wait_for_port(host=host, port=22, max_attempts=20, wait_for=0.1) +def wait_for_ssh(host, max_attempts=120, wait_for=0.5): + return wait_for_port(host=host, port=22, max_attempts=max_attempts, wait_for=wait_for) def wait_for_port(host, port, max_attempts=20, wait_for=0.1): for i in range(max_attempts): From 8ef6c0762577d212148f3fa84ef181dec1859bfd Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Fri, 20 Aug 2021 07:53:57 +0100 Subject: [PATCH 19/35] Using Python_Logger in utlis.Status --- osbot_utils/utils/Status.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Status.py b/osbot_utils/utils/Status.py index 627673c3..70e09e5e 100644 --- a/osbot_utils/utils/Status.py +++ b/osbot_utils/utils/Status.py @@ -1,8 +1,9 @@ import logging # todo refactor into Status class +from osbot_utils.utils.Python_Logger import Python_Logger -logger = logging.getLogger() +logger = Python_Logger().setup() # logging.getLogger() # todo: def status_message(status, message:str=None, data=None, error=None): return { 'data' : data , diff --git a/setup.py b/setup.py index 56d96226..2d08fcf5 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - version = "0.7.0" , # change this on every release + version = "0.7.1" , # change this on every release name = "osbot_utils" , author = "Dinis Cruz", author_email = "dinis.cruz@owasp.org", From 4a483772fa2ac190df3af399344a9f9c6b12935f Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Fri, 20 Aug 2021 23:04:26 +0100 Subject: [PATCH 20/35] changed logging statement --- osbot_utils/utils/Http.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Http.py b/osbot_utils/utils/Http.py index 470a3139..60b114aa 100644 --- a/osbot_utils/utils/Http.py +++ b/osbot_utils/utils/Http.py @@ -28,7 +28,7 @@ def port_is_open(port : int , host='0.0.0.0', timeout=1.0): result = sock.connect_ex((host, port)) return result == 0 except: - logger.exception(f'port {port} was closed in server {host}') + logger.error(f'port {port} was closed in server {host}') return False diff --git a/setup.py b/setup.py index 2d08fcf5..7f78c635 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - version = "0.7.1" , # change this on every release + version = "0.7.2" , # change this on every release name = "osbot_utils" , author = "Dinis Cruz", author_email = "dinis.cruz@owasp.org", From 7cb7858811c3e07107aac41a7722459ea7529351 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sat, 21 Aug 2021 12:50:06 +0100 Subject: [PATCH 21/35] minor fix to http logging --- osbot_utils/utils/Http.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/osbot_utils/utils/Http.py b/osbot_utils/utils/Http.py index 60b114aa..c413ba1b 100644 --- a/osbot_utils/utils/Http.py +++ b/osbot_utils/utils/Http.py @@ -18,17 +18,18 @@ def current_host_online(url_to_use='http://www.google.com'): def dns_ip_address(host): return socket.gethostbyname(host) -def is_port_open(host, port, timeout=0.5): +def is_port_open(host, port, timeout=0.5, log_error=True): return port_is_open(host=host, port=port, timeout=timeout) -def port_is_open(port : int , host='0.0.0.0', timeout=1.0): +def port_is_open(port : int , host='0.0.0.0', timeout=1.0, log_error=True): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) result = sock.connect_ex((host, port)) return result == 0 except: - logger.error(f'port {port} was closed in server {host}') + if log_error: + logger.error(f'port {port} was closed in server {host}') return False @@ -62,7 +63,7 @@ def wait_for_ssh(host, max_attempts=120, wait_for=0.5): def wait_for_port(host, port, max_attempts=20, wait_for=0.1): for i in range(max_attempts): - if is_port_open(host=host,port=port,timeout=wait_for): + if is_port_open(host=host,port=port,timeout=wait_for, log_error=False): return True return False From 28d8a24a144e19cdbd37d26790b2f7b0bebe5e32 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 22 Aug 2021 17:36:21 +0100 Subject: [PATCH 22/35] Improve cache_on_self to support caching based on params values --- .../decorators/methods/cache_on_self.py | 44 +++++++- osbot_utils/utils/Misc.py | 6 ++ .../decorators/methods/test_cache_on_self.py | 100 ++++++++++++++++-- 3 files changed, 138 insertions(+), 12 deletions(-) diff --git a/osbot_utils/decorators/methods/cache_on_self.py b/osbot_utils/decorators/methods/cache_on_self.py index 3dc8c215..4de5e1cb 100644 --- a/osbot_utils/decorators/methods/cache_on_self.py +++ b/osbot_utils/decorators/methods/cache_on_self.py @@ -1,6 +1,18 @@ import inspect from functools import wraps +from osbot_utils.utils.Misc import str_md5 + +#from osbot_utils.utils.Dev import pprint + +CACHE_ON_SELF_KEY_PREFIX = 'cache_on_self' +CACHE_ON_SELF_TYPES = [int, float, bytearray, bytes, bool, + complex, str] + +# not supported for now (need to understand side effect, ) +# - set, dict, range,, tuple, list : cloud have inner objects +# - memoryview : returns unique memory location value + def cache_on_self(function): """ @@ -12,8 +24,36 @@ def wrapper(*args, **kwargs): raise Exception("In Method_Wrappers.cache_on_self could not find self") self = args[0] # get self - cache_id = f'osbot_cache_return_value__{function.__name__}' # generate cache_id + cache_id = cache_on_self_get_cache_in_key(function, args, kwargs) if hasattr(self, cache_id) is False: # check if return_value has been set setattr(self, cache_id, function(*args, **kwargs)) # invoke function and capture the return value return getattr(self, cache_id) # return the return value - return wrapper \ No newline at end of file + return wrapper + +def cache_on_self_args_to_str(args): + args_values_as_str = '' + if args: + for arg in args: + if type(arg) in CACHE_ON_SELF_TYPES: + args_values_as_str += str(arg) + return args_values_as_str + + +def cache_on_self_get_cache_in_key(function, args=None, kwargs=None): + key_name = function.__name__ + args_md5 = '' + kwargs_md5 = '' + args_values_as_str = cache_on_self_args_to_str(args) + if args_values_as_str: + args_md5 = str_md5(args_values_as_str) + return f'{CACHE_ON_SELF_KEY_PREFIX}_{key_name}_{args_md5}_{kwargs_md5}' + + class_name = self_obj.__class__.__name__ + + function_name = function_obj.__name__ + if params: + params_as_string = '_'.join(str(x) for x in params).replace('/',' ') + params_md5 = str_md5(params_as_string) + return f'{class_name}_{function_name}_{params_md5}.gz' + else: + return f'{class_name}_{function_name}.gz' \ No newline at end of file diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 13ffd619..c1e92e1f 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -269,6 +269,12 @@ def lower(target : str): return target.lower() return "" +def obj_data(target=None): + data = {} + for key,value in obj_items(target): + data[key] = value + return data + def obj_dict(target=None): if target and hasattr(target,'__dict__'): return target.__dict__ diff --git a/tests/decorators/methods/test_cache_on_self.py b/tests/decorators/methods/test_cache_on_self.py index 594cbf27..292963aa 100644 --- a/tests/decorators/methods/test_cache_on_self.py +++ b/tests/decorators/methods/test_cache_on_self.py @@ -1,24 +1,104 @@ from pprint import pprint from unittest import TestCase -from osbot_utils.decorators.methods.cache_on_self import cache_on_self +from osbot_utils.decorators.methods.cache_on_self import cache_on_self, cache_on_self_get_cache_in_key, \ + CACHE_ON_SELF_KEY_PREFIX, cache_on_self_args_to_str from osbot_utils.testing.Catch import Catch +from osbot_utils.utils.Misc import obj_data +class An_Class: + @cache_on_self + def an_function(self): + return 42 + + @cache_on_self + def echo(self, value): + return value + + @cache_on_self + def echo_args(self, *args): + return args + class test_cache_on_self(TestCase): def test_cache_on_self(self): - class An_Class: - @cache_on_self - def an_function(self): - return 42 + an_class_1 = An_Class() # create 1st instance + cache_key = cache_on_self_get_cache_in_key(an_class_1.an_function) # get key from self + assert cache_key == f'{CACHE_ON_SELF_KEY_PREFIX}_an_function__' # confirm cache key value + assert obj_data(an_class_1) == {} # confirm cache key has not been added to self + + # testing function that returns static value + assert an_class_1.an_function() == 42 # invoke method, set cache and confirm return value + assert obj_data(an_class_1) == { cache_key : 42} # confirm attribute has been set in class + pprint(obj_data(an_class_1)) + assert an_class_1.cache_on_self_an_function__ == 42 # which can be accessed directly + an_class_1.cache_on_self_an_function__ = 12 # if we change the attribute directly + assert obj_data(an_class_1) == { cache_key : 12} # confirm value changes (via obj data) + assert an_class_1.cache_on_self_an_function__ == 12 # confirm value change (directly) + + an_class_2 = An_Class() # create 2nd instance + assert an_class_2.an_function() == 42 # confirm previous version was not affected + + an_class_3 = An_Class() # create 3rd instance + assert an_class_3.an_function() == 42 # confirm previous version was not affected + + # testing function that returns dynamic value (with args) + assert an_class_1.echo(111) == 111 # confirm returns echo value + assert an_class_1.echo(111) == 111 + assert an_class_1.echo(222) == 222 # config, new value has been set + assert an_class_1.echo(111) == 111 + + assert an_class_2.echo(333) == 333 # confirm returns echo value + assert an_class_2.echo(333) == 333 + assert an_class_2.echo(444) == 444 # config, new value has been set + + assert an_class_3.echo(555) == 555 # confirm returns echo value + assert an_class_3.echo(555) == 555 + assert an_class_3.echo(666) == 666 # config, new value has been set + + assert obj_data(an_class_1) == {'cache_on_self_an_function__': 12, + 'cache_on_self_echo_698d51a19d8a121ce581499d7b701668_': 111, + 'cache_on_self_echo_bcbe3365e6ac95ea2c0343a2395834dd_': 222 } + + assert obj_data(an_class_2) == {'cache_on_self_an_function__': 42, + 'cache_on_self_echo_310dcbbf4cce62f762a2aaa148d556bd_': 333, + 'cache_on_self_echo_550a141f12de6341fba65b0ad0433500_': 444} + + assert obj_data(an_class_3) == {'cache_on_self_an_function__': 42, + 'cache_on_self_echo_15de21c670ae7c3f6f3f1f37029303c9_': 555, + 'cache_on_self_echo_fae0b27c451c728867a567e8c1bb4e53_': 666} + + # testing function that returns dynamic value (with kargs) + assert an_class_1.echo(value=111) == 111 # confirm returns echo value + assert an_class_1.echo(value=222) == 111 # bug, should had returned 222 + + assert an_class_2.echo(value=333) == 333 # confirm returns echo value + assert an_class_2.echo(value=444) == 333 # bug, should had returned 444 + + assert an_class_3.echo(value=555) == 555 # confirm returns echo value + assert an_class_3.echo(value=666) == 555 # bug, should had returned 666 + + def test_cache_on_self__multiple_types_in_arg_cache(self): + args = ('a', 1, 1.0) + an_class = An_Class() + assert an_class.echo_args(*args) == args + assert cache_on_self_args_to_str(args) == "a11.0" + + args = ('a', None, 'bbb', [], {}) + assert an_class.echo_args(*args) == args + assert cache_on_self_args_to_str(args) == "abbb" + + args = ('a', -1, ['a'], {'b':None}) + assert an_class.echo_args(*args) == args + assert cache_on_self_args_to_str(args) == "a-1" + + args = (1, int(1), float(1), bytearray(b'1'), bytes(b'1'), bool(True), complex(1), str('1')) + assert an_class.echo_args(*args) == args + assert an_class.echo_args(*args) == (1, 1, 1.0, bytearray(b'1'), b'1', True, (1 + 0j), '1') + assert cache_on_self_args_to_str(args) == "111.0bytearray(b'1')b'1'True(1+0j)1" - an_class = An_Class() - assert an_class.an_function() == 42 - assert an_class.osbot_cache_return_value__an_function == 42 - an_class.osbot_cache_return_value__an_function = 12 - assert an_class.osbot_cache_return_value__an_function == 12 def test_cache_on_self__outside_an_class(self): From 27f620b1d209ca357058d0792dd5b70fa7b15032 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Tue, 24 Aug 2021 10:30:48 +0100 Subject: [PATCH 23/35] improved cache_on_self --- .../decorators/methods/cache_on_self.py | 32 ++++++++++++------- .../decorators/methods/test_cache_on_self.py | 25 +++++++++------ 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/osbot_utils/decorators/methods/cache_on_self.py b/osbot_utils/decorators/methods/cache_on_self.py index 4de5e1cb..fe4b73be 100644 --- a/osbot_utils/decorators/methods/cache_on_self.py +++ b/osbot_utils/decorators/methods/cache_on_self.py @@ -30,7 +30,7 @@ def wrapper(*args, **kwargs): return getattr(self, cache_id) # return the return value return wrapper -def cache_on_self_args_to_str(args): +def cache_on_self__args_to_str(args): args_values_as_str = '' if args: for arg in args: @@ -38,22 +38,32 @@ def cache_on_self_args_to_str(args): args_values_as_str += str(arg) return args_values_as_str +def cache_on_self__kwargs_to_str(kwargs): + kwargs_values_as_str = '' + if kwargs: + for key,value in kwargs.items(): + if type(value) in CACHE_ON_SELF_TYPES: + kwargs_values_as_str += f'{key}:{value}|' + return kwargs_values_as_str def cache_on_self_get_cache_in_key(function, args=None, kwargs=None): key_name = function.__name__ args_md5 = '' kwargs_md5 = '' - args_values_as_str = cache_on_self_args_to_str(args) + args_values_as_str = cache_on_self__args_to_str(args) + kwargs_values_as_str = cache_on_self__kwargs_to_str(kwargs) if args_values_as_str: args_md5 = str_md5(args_values_as_str) + if kwargs_values_as_str: + kwargs_md5 = str_md5(kwargs_values_as_str) return f'{CACHE_ON_SELF_KEY_PREFIX}_{key_name}_{args_md5}_{kwargs_md5}' - class_name = self_obj.__class__.__name__ - - function_name = function_obj.__name__ - if params: - params_as_string = '_'.join(str(x) for x in params).replace('/',' ') - params_md5 = str_md5(params_as_string) - return f'{class_name}_{function_name}_{params_md5}.gz' - else: - return f'{class_name}_{function_name}.gz' \ No newline at end of file + # class_name = self_obj.__class__.__name__ + # + # function_name = function_obj.__name__ + # if params: + # params_as_string = '_'.join(str(x) for x in params).replace('/',' ') + # params_md5 = str_md5(params_as_string) + # return f'{class_name}_{function_name}_{params_md5}.gz' + # else: + # return f'{class_name}_{function_name}.gz' \ No newline at end of file diff --git a/tests/decorators/methods/test_cache_on_self.py b/tests/decorators/methods/test_cache_on_self.py index 292963aa..c09202ca 100644 --- a/tests/decorators/methods/test_cache_on_self.py +++ b/tests/decorators/methods/test_cache_on_self.py @@ -1,8 +1,9 @@ -from pprint import pprint from unittest import TestCase +from osbot_utils.utils.Dev import pprint + from osbot_utils.decorators.methods.cache_on_self import cache_on_self, cache_on_self_get_cache_in_key, \ - CACHE_ON_SELF_KEY_PREFIX, cache_on_self_args_to_str + CACHE_ON_SELF_KEY_PREFIX, cache_on_self__args_to_str, cache_on_self__kwargs_to_str from osbot_utils.testing.Catch import Catch from osbot_utils.utils.Misc import obj_data @@ -72,32 +73,38 @@ def test_cache_on_self(self): # testing function that returns dynamic value (with kargs) assert an_class_1.echo(value=111) == 111 # confirm returns echo value - assert an_class_1.echo(value=222) == 111 # bug, should had returned 222 + assert an_class_1.echo(value=222) == 222 # confirms new value assert an_class_2.echo(value=333) == 333 # confirm returns echo value - assert an_class_2.echo(value=444) == 333 # bug, should had returned 444 + assert an_class_2.echo(value=444) == 444 # confirms new value assert an_class_3.echo(value=555) == 555 # confirm returns echo value - assert an_class_3.echo(value=666) == 555 # bug, should had returned 666 + assert an_class_3.echo(value=666) == 666 # confirms new value def test_cache_on_self__multiple_types_in_arg_cache(self): args = ('a', 1, 1.0) an_class = An_Class() assert an_class.echo_args(*args) == args - assert cache_on_self_args_to_str(args) == "a11.0" + assert cache_on_self__args_to_str(args) == "a11.0" args = ('a', None, 'bbb', [], {}) assert an_class.echo_args(*args) == args - assert cache_on_self_args_to_str(args) == "abbb" + assert cache_on_self__args_to_str(args) == "abbb" args = ('a', -1, ['a'], {'b':None}) assert an_class.echo_args(*args) == args - assert cache_on_self_args_to_str(args) == "a-1" + assert cache_on_self__args_to_str(args) == "a-1" args = (1, int(1), float(1), bytearray(b'1'), bytes(b'1'), bool(True), complex(1), str('1')) assert an_class.echo_args(*args) == args assert an_class.echo_args(*args) == (1, 1, 1.0, bytearray(b'1'), b'1', True, (1 + 0j), '1') - assert cache_on_self_args_to_str(args) == "111.0bytearray(b'1')b'1'True(1+0j)1" + assert cache_on_self__args_to_str(args) == "111.0bytearray(b'1')b'1'True(1+0j)1" + + def test_cache_on_self__kwargs_to_str(self): + assert cache_on_self__kwargs_to_str({"an":"value" }) == 'an:value|' + assert cache_on_self__kwargs_to_str({"a": "b","c":"d"}) == 'a:b|c:d|' + assert cache_on_self__kwargs_to_str({"an": None }) == '' + assert cache_on_self__kwargs_to_str({"an": 1 }) == 'an:1|' def test_cache_on_self__outside_an_class(self): From f8e179984abaa89e04b73fae503bcd223db3d8e4 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Thu, 26 Aug 2021 11:24:11 +0100 Subject: [PATCH 24/35] bumped version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f78c635..5e6ca775 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - version = "0.7.2" , # change this on every release + version = "0.7.3" , # change this on every release name = "osbot_utils" , author = "Dinis Cruz", author_email = "dinis.cruz@owasp.org", From c6e42443df64fd84a7f63ada6abc019bfdcb3d90 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Thu, 26 Aug 2021 14:57:56 +0100 Subject: [PATCH 25/35] don't log errors by default on the Http.port_is_open method --- osbot_utils/utils/Http.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Http.py b/osbot_utils/utils/Http.py index c413ba1b..01471155 100644 --- a/osbot_utils/utils/Http.py +++ b/osbot_utils/utils/Http.py @@ -21,7 +21,7 @@ def dns_ip_address(host): def is_port_open(host, port, timeout=0.5, log_error=True): return port_is_open(host=host, port=port, timeout=timeout) -def port_is_open(port : int , host='0.0.0.0', timeout=1.0, log_error=True): +def port_is_open(port : int , host='0.0.0.0', timeout=1.0, log_error=False): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) diff --git a/setup.py b/setup.py index 7f78c635..5e6ca775 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - version = "0.7.2" , # change this on every release + version = "0.7.3" , # change this on every release name = "osbot_utils" , author = "Dinis Cruz", author_email = "dinis.cruz@owasp.org", From e372eed1771ab495be8ba5484dfe2016ab0df321 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Fri, 27 Aug 2021 08:40:35 +0100 Subject: [PATCH 26/35] minor refactoring --- osbot_utils/decorators/methods/cache_on_self.py | 4 ++-- setup.py | 2 +- tests/decorators/methods/test_cache_on_self.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/osbot_utils/decorators/methods/cache_on_self.py b/osbot_utils/decorators/methods/cache_on_self.py index fe4b73be..3432fddc 100644 --- a/osbot_utils/decorators/methods/cache_on_self.py +++ b/osbot_utils/decorators/methods/cache_on_self.py @@ -24,7 +24,7 @@ def wrapper(*args, **kwargs): raise Exception("In Method_Wrappers.cache_on_self could not find self") self = args[0] # get self - cache_id = cache_on_self_get_cache_in_key(function, args, kwargs) + cache_id = cache_on_self__get_cache_in_key(function, args, kwargs) if hasattr(self, cache_id) is False: # check if return_value has been set setattr(self, cache_id, function(*args, **kwargs)) # invoke function and capture the return value return getattr(self, cache_id) # return the return value @@ -46,7 +46,7 @@ def cache_on_self__kwargs_to_str(kwargs): kwargs_values_as_str += f'{key}:{value}|' return kwargs_values_as_str -def cache_on_self_get_cache_in_key(function, args=None, kwargs=None): +def cache_on_self__get_cache_in_key(function, args=None, kwargs=None): key_name = function.__name__ args_md5 = '' kwargs_md5 = '' diff --git a/setup.py b/setup.py index 5e6ca775..08d6a069 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - version = "0.7.3" , # change this on every release + version = "0.7.4" , # change this on every release name = "osbot_utils" , author = "Dinis Cruz", author_email = "dinis.cruz@owasp.org", diff --git a/tests/decorators/methods/test_cache_on_self.py b/tests/decorators/methods/test_cache_on_self.py index c09202ca..588ddd67 100644 --- a/tests/decorators/methods/test_cache_on_self.py +++ b/tests/decorators/methods/test_cache_on_self.py @@ -2,7 +2,7 @@ from osbot_utils.utils.Dev import pprint -from osbot_utils.decorators.methods.cache_on_self import cache_on_self, cache_on_self_get_cache_in_key, \ +from osbot_utils.decorators.methods.cache_on_self import cache_on_self, cache_on_self__get_cache_in_key, \ CACHE_ON_SELF_KEY_PREFIX, cache_on_self__args_to_str, cache_on_self__kwargs_to_str from osbot_utils.testing.Catch import Catch from osbot_utils.utils.Misc import obj_data @@ -26,7 +26,7 @@ class test_cache_on_self(TestCase): def test_cache_on_self(self): an_class_1 = An_Class() # create 1st instance - cache_key = cache_on_self_get_cache_in_key(an_class_1.an_function) # get key from self + cache_key = cache_on_self__get_cache_in_key(an_class_1.an_function) # get key from self assert cache_key == f'{CACHE_ON_SELF_KEY_PREFIX}_an_function__' # confirm cache key value assert obj_data(an_class_1) == {} # confirm cache key has not been added to self From 507a8ab8c3b36e68e3cd7d2f87f74a7fc1ec4f23 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sat, 28 Aug 2021 13:20:12 +0100 Subject: [PATCH 27/35] added multiple file utils --- .../decorators/methods/remove_return_value.py | 2 +- osbot_utils/utils/Files.py | 51 ++++++++++++++++++- osbot_utils/utils/Misc.py | 22 ++++++-- tests/utils/test_Files.py | 7 +-- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/osbot_utils/decorators/methods/remove_return_value.py b/osbot_utils/decorators/methods/remove_return_value.py index 8c4cb135..9eac98a6 100644 --- a/osbot_utils/decorators/methods/remove_return_value.py +++ b/osbot_utils/decorators/methods/remove_return_value.py @@ -19,4 +19,4 @@ def wrapper(*args,**kwargs): # wrapper function return wrapper # return wrapper function #todo: check usages and remove legacy method -remove = remove_return_value \ No newline at end of file +remove = remove_return_value \ No newline at end of file diff --git a/osbot_utils/utils/Files.py b/osbot_utils/utils/Files.py index cb6a96d0..97311807 100644 --- a/osbot_utils/utils/Files.py +++ b/osbot_utils/utils/Files.py @@ -137,8 +137,37 @@ def folder_not_exists(path): return folder_exists(path) is False @staticmethod - def path_combine(path1, path2): - return abspath(join(path1, path2)) + def folder_sub_folders(path): + result = [] + item: os.DirEntry + if Files.is_folder(path): + for item in os.scandir(path): + if item.is_dir(): + result.append(item.path) + return result + + @staticmethod + def folders_names(folders : list): + result = [] + for folder in folders: + if Files.is_folder(folder): + result.append(Files.file_name(folder)) + return result + + @staticmethod + def folders_sub_folders(folders : list): + result = [] + for folder in folders: + result.extend(Files.folder_sub_folders(folder)) + return result + + @staticmethod + def is_file(target): + return os.path.isfile(target) + + @staticmethod + def is_folder(target): + return os.path.isdir(target) @staticmethod def lines(path): @@ -168,6 +197,10 @@ def open_gz(path, mode='r'): def open_bytes(path): return Files.open(path, mode='rb') + @staticmethod + def path_combine(path1, path2): + return abspath(join(path1, path2)) + @staticmethod def parent_folder(path): return os.path.dirname(path) @@ -182,6 +215,13 @@ def save(contents, path=None, extension=None): file_create(path, contents) return path + @staticmethod + def sub_folders(target): + if type(target) is list: + return Files.folders_sub_folders(target) + if type(target) is str: + return Files.folder_sub_folders(target) + return [] @staticmethod def save_bytes_as_file(bytes_to_save, path=None, extension=None): @@ -328,8 +368,14 @@ def zip_files(base_folder, file_pattern="*.*", target_file=None): folder_name = Files.folder_name folder_temp = Files.temp_folder folder_files = Files.files +folder_sub_folders = Files.folder_sub_folders folder_zip = Files.zip_folder +folders_names = Files.folders_names + +is_file = Files.is_file +is_folder = Files.is_folder + load_file = Files.contents load_file_gz = Files.contents_gz @@ -341,6 +387,7 @@ def zip_files(base_folder, file_pattern="*.*", target_file=None): save_bytes_as_file = Files.save_bytes_as_file save_string_as_file = Files.save +sub_folders = Files.sub_folders temp_file = Files.temp_file temp_filename = Files.temp_filename diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 99def553..e34d14ea 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -1,5 +1,6 @@ import base64 import hashlib +import inspect import logging import os import random @@ -266,6 +267,9 @@ def str_index(target:str, source:str): except: return -1 +def sys_path_python(python_folder='lib/python'): + return list_contains(sys.path, python_folder) + def str_md5(text : str): if text: return bytes_md5(text.encode()) @@ -280,6 +284,17 @@ def none_or_empty(target,field): def print_date_now(use_utc=True): print(date_time_now(use_utc=use_utc)) +def print_object_members(target, max_width=120, show_internals=False): + print() + print(f"Members for object: {target}"[:max_width]) + print() + print(f"{'field':<20} | value") + print(f"{'-' * max_width}") + for name, val in inspect.getmembers(target): + if not show_internals and name.startswith("__"): + continue + print(f"{name:<20} | {val}"[:max_width]) + def print_time_now(use_utc=True): print(time_now(use_utc=use_utc)) @@ -359,13 +374,13 @@ def random_text(prefix=None,length=12): def random_uuid(): return str(uuid.uuid4()) -def remove(target_string, string_to_remove): +def remove(target_string, string_to_remove): # todo: refactor to str_* return replace(target_string, string_to_remove, '') -def remove_multiple_spaces(target): +def remove_multiple_spaces(target): # todo: refactor to str_* return re.sub(' +', ' ', target) -def replace(target_string, string_to_find, string_to_replace): +def replace(target_string, string_to_find, string_to_replace): # todo: refactor to str_* return target_string.replace(string_to_find, string_to_replace) def remove_html_tags(html): @@ -434,6 +449,7 @@ def word_wrap_escaped(text,length = 40): list_contains = list_filter_contains new_guid = random_uuid str_lines = split_lines +str_remove = remove random_id = random_string wait_for = wait diff --git a/tests/utils/test_Files.py b/tests/utils/test_Files.py index 8d36045c..261c57f8 100644 --- a/tests/utils/test_Files.py +++ b/tests/utils/test_Files.py @@ -1,11 +1,12 @@ from unittest import TestCase +from osbot_utils.utils.Dev import pprint from osbot_utils.utils.Files import Files, path_combine, parent_folder, path_current, save_bytes_as_file, file_bytes, \ temp_file, file_create, file_delete, file_exists, file_contents, file_copy, file_contents_as_bytes, file_name, \ folder_name, folder_files, file_not_exists, temp_folder, folder_copy, path_append, folder_exists, folder_create, \ folder_delete_all, folder_not_exists, temp_folder_with_temp_file, folder_zip, file_unzip, file_extension, \ zip_file_list, zip_files, save_string_as_file, file_write_gz, file_contents_gz, file_size, file_write, file_find, \ - file_lines, file_create_gz, file_lines_gz, parent_folder_combine, file_write_bytes, file_open_bytes + file_lines, file_create_gz, file_lines_gz, parent_folder_combine, file_write_bytes, file_open_bytes, sub_folders from osbot_utils.utils.Misc import random_bytes, random_string, remove @@ -169,8 +170,6 @@ def test_folder_files(self): assert path_combine(folder, 'test_Files.py') in folder_files(folder) assert path_combine(folder, 'test_Json.py' ) in folder_files(folder) - - def test_folder_zip(self): folder = temp_folder_with_temp_file(file_contents=random_string()) print() @@ -200,6 +199,8 @@ def test_save_string_as_file(self): temp_file = save_string_as_file(data) assert file_contents(temp_file) == data + def test_sub_folders(self): + assert '/usr/bin' in sub_folders('/usr') def test_temp_folder(self): assert Files.exists(Files.temp_folder()) From b1b850442fe34de6b921b23d3f0fa2af2e537bc4 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sat, 28 Aug 2021 13:37:56 +0100 Subject: [PATCH 28/35] minor test fix --- setup.py | 2 +- tests/testing/test_Hook_Method.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index af5c5fc7..aaefd3ae 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - version = "0.7.5" , # change this on every release + version = "0.7.6" , # change this on every release name = "osbot_utils" , author = "Dinis Cruz", author_email = "dinis.cruz@owasp.org", diff --git a/tests/testing/test_Hook_Method.py b/tests/testing/test_Hook_Method.py index 074376f1..3ba1c3ec 100644 --- a/tests/testing/test_Hook_Method.py +++ b/tests/testing/test_Hook_Method.py @@ -99,7 +99,7 @@ def test_wrap__unwrap(self): # todo: refactor this to use a c assert self.wrap_method.calls[2]['kwargs' ] == {'allow_redirects': False} assert self.wrap_method.calls[2]['return_value'].status_code == 200 assert self.wrap_method.calls[3]['args' ] == ('get', 'https://www.google.com/404') - assert self.wrap_method.calls[3]['kwargs' ] == {'params': None} + assert self.wrap_method.calls[3]['kwargs' ] == {'allow_redirects': True, 'params': None} assert self.wrap_method.calls[3]['return_value'].status_code == 404 self.wrap_method.unwrap() From e608be165ceb04e596f24b39793cadbad224cb61 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 29 Aug 2021 16:56:01 +0100 Subject: [PATCH 29/35] added class_functions method --- osbot_utils/utils/Misc.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 6d3d355a..5a5dd760 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -49,6 +49,15 @@ def class_name(target): if target: return type(target).__name__ +def class_functions(target): + functions = {} + for function_name, function_ref in inspect.getmembers(type(target), predicate=inspect.isfunction): + functions[function_name] = function_ref + return functions + +def class_functions_names(target): + return list_set(class_functions(target)) + def convert_to_number(value): if value: try: @@ -172,6 +181,9 @@ def list_find(array:list, item): return array.index(item) return -1 +def list_get_field(values, field): + return [item.get(field) for item in values] + def list_index_by(values, index_by): from osbot_utils.fluent.Fluent_Dict import Fluent_Dict results = {} @@ -282,10 +294,10 @@ def obj_dict(target=None): return {} def obj_items(target=None): - return list(obj_dict(target).items()) + return sorted(list(obj_dict(target).items())) def obj_keys(target=None): - return list(obj_dict(target).keys()) + return sorted(list(obj_dict(target).keys())) def obj_get_value(target=None, key=None, default=None): return get_field(target=target, field=key, default=default) @@ -497,6 +509,7 @@ def word_wrap_escaped(text,length = 40): datetime_now = date_time_now list_contains = list_filter_contains new_guid = random_uuid +obj_list_set = obj_keys str_lines = split_lines str_remove = remove random_id = random_string From 69661863946283012a5157a969d397c72a46e823 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Sun, 29 Aug 2021 17:53:44 +0100 Subject: [PATCH 30/35] added yaml_dump method --- osbot_utils/utils/Yaml.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/osbot_utils/utils/Yaml.py b/osbot_utils/utils/Yaml.py index cdb48a14..8b06b1f3 100644 --- a/osbot_utils/utils/Yaml.py +++ b/osbot_utils/utils/Yaml.py @@ -4,6 +4,10 @@ class Yaml: + @staticmethod + def yaml_dump(python_object): + return yaml.dump(python_object) + @staticmethod def yaml_load(path_file): with open(path_file) as file: @@ -14,14 +18,20 @@ def yaml_parse(yaml_code): return yaml.safe_load(yaml_code) @staticmethod - def yaml_save(yaml_code, path_file=None): #todo: refactor other 'save' methods to have the content as the first param (to allow for creation of temp files when path_file path is not provided) + def yaml_save(python_object, path_file=None): #todo: refactor other 'save' methods to have the content as the first param (to allow for creation of temp files when path_file path is not provided) if path_file is None: path_file = temp_file(extension=".yaml") with open(path_file, 'w') as file: - yaml.dump(yaml_code, file) + yaml.dump(python_object, file) return path_file # also add these methods to the top level to allow direct import and usage (without the Yaml.*) -yaml_load = Yaml.yaml_load -yaml_parse = Yaml.yaml_parse -yaml_save = Yaml.yaml_save \ No newline at end of file + +yaml_dump = Yaml.yaml_dump +yaml_load = Yaml.yaml_load +yaml_parse = Yaml.yaml_parse +yaml_save = Yaml.yaml_save + +yaml_to_str = yaml_dump +yaml_to_json = yaml_parse +str_to_yaml = yaml_load \ No newline at end of file From d9a6e2e0255287f868a000285ed226e2ee588f6c Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Tue, 7 Sep 2021 10:17:45 +0100 Subject: [PATCH 31/35] minor fix and added new Misc method list_sorted --- osbot_utils/utils/Misc.py | 8 ++++++-- setup.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 5a5dd760..0cccdfe1 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -187,8 +187,9 @@ def list_get_field(values, field): def list_index_by(values, index_by): from osbot_utils.fluent.Fluent_Dict import Fluent_Dict results = {} - for item in values: - results[item.get(index_by)] = item + if values and index_by: + for item in values: + results[item.get(index_by)] = item return Fluent_Dict(results) def list_group_by(values, group_by): @@ -231,6 +232,9 @@ def list_set_dict(target): def list_filter(target_list, filter_function): return list(filter(filter_function, target_list)) +def list_sorted(target_list, key, descending=False): + return list(sorted(target_list, key= lambda x:x.get(key,None) ,reverse=descending)) + def list_filter_starts_with(target_list, prefix): return list_filter(target_list, lambda x: x.startswith(prefix)) diff --git a/setup.py b/setup.py index aaefd3ae..9c5f89fb 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ long_description = fh.read() setuptools.setup( - version = "0.7.6" , # change this on every release + version = "0.7.7" , # change this on every release name = "osbot_utils" , author = "Dinis Cruz", author_email = "dinis.cruz@owasp.org", From 26cb38e3d1a66e9b7331763c57a6e846293902c9 Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Tue, 7 Sep 2021 13:15:11 +0100 Subject: [PATCH 32/35] added list_zip method to Misc class --- osbot_utils/utils/Misc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 0cccdfe1..e6d54cb0 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -224,7 +224,13 @@ def list_pop_and_trim(array, position=None): return value def list_set(target): - return sorted(list(set(target))) + if target: + return sorted(list(set(target))) + return [] + +def list_zip(*args): + return list(zip(*args)) + def list_set_dict(target): return sorted(list(set(obj_dict(target)))) From 7269390b8fc42a38b8318a21cb824d63eca88bed Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Tue, 12 Oct 2021 17:50:14 +0100 Subject: [PATCH 33/35] added method list_contains_list --- osbot_utils/utils/Misc.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index e6d54cb0..997d8639 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -176,6 +176,15 @@ def list_add(array : list, value): array.append(value) return value +def list_contains_list(array : list, values): + if array is not None: + if type(values) is list: + for item in values: + if item in array is False: + return False + return True + return False + def list_find(array:list, item): if item in array: return array.index(item) From 2c39e7dcecb7ddc6cb8dd41668fc93dfaf66fe3f Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Thu, 4 Nov 2021 02:34:55 +0000 Subject: [PATCH 34/35] added method stub --- osbot_utils/utils/Json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/osbot_utils/utils/Json.py b/osbot_utils/utils/Json.py index 21affeb6..fbb6dbad 100644 --- a/osbot_utils/utils/Json.py +++ b/osbot_utils/utils/Json.py @@ -98,6 +98,7 @@ def save_file_pretty_gz(python_object, path=None): def json_save_tmp_file(python_object, pretty=True): return Json.save_file(python_object=python_object, pretty=pretty, path=None) +json_dump = Json.dumps json_dumps = Json.dumps json_format = Json.dumps json_load_file = Json.load_file From 1ec0ba509cb5fd95a543d5cf9af5dc37dae40b5c Mon Sep 17 00:00:00 2001 From: Dinis Cruz Date: Mon, 13 Dec 2021 10:43:52 +0000 Subject: [PATCH 35/35] Minor refactoring --- osbot_utils/utils/Misc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index 997d8639..d875a52b 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -428,15 +428,15 @@ def random_password(length=24, prefix=''): password = password.replace(item, '_') return password -def random_string(length=8,prefix=''): +def random_string(length:int=8,prefix:str=''): return prefix + ''.join(random.choices(string.ascii_uppercase, k=length)).lower() -def random_string_and_numbers(length=6,prefix=''): +def random_string_and_numbers(length:int=6,prefix:str=''): return prefix + ''.join(random.choices(string.ascii_uppercase + string.digits, k=length)) -def random_text(prefix=None,length=12): +def random_text(prefix:str=None,length:int=12): if prefix is None: prefix = 'text_' - if last_letter(prefix) != '_': + if last_letter(prefix) not in ['_','/']: prefix += '_' return random_string_and_numbers(length=length, prefix=prefix)