From d47e8d2bb0befbbb55b1ef18e7289d3dfd1f811a Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 4 Jul 2023 16:54:42 +0700 Subject: [PATCH 1/8] add minio --- py_utility/minio_.py | 78 ++++++++++++++++++ py_utility/progress.py | 182 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 py_utility/minio_.py create mode 100644 py_utility/progress.py diff --git a/py_utility/minio_.py b/py_utility/minio_.py new file mode 100644 index 0000000..72a1261 --- /dev/null +++ b/py_utility/minio_.py @@ -0,0 +1,78 @@ +from typing import List +from minio import Minio +from minio.error import S3Error +import os +from tqdm import tqdm +from py_utility.progress import Progress + +def get_all_file_paths(directory): + directory = os.path.abspath(directory) + file_paths = [] + + for root, directories, files in os.walk(directory): + for filename in files: + filepath = os.path.join(root, filename) + file_paths.append(filepath) + file_paths_ = [file[len(directory)+1:] for file in file_paths] + return file_paths, file_paths_ + +class MinioClient(Minio): + def __init__(self, endpoint, access_key=None, secret_key=None, session_token=None, secure=True, region=None, http_client=None, credentials=None, cert_check=True): + super().__init__(endpoint, access_key, secret_key, session_token, secure, region, http_client, credentials, cert_check) + + def upload(self,bucket_name: str,prefix:str, path_local_upload:str): + """ + Uploads a file or directory to a Minio bucket. + + Args: + bucket_name (str): The name of the Minio bucket. + prefix (str): The prefix to be added to the file(s) in the bucket. + path_local_upload (str): The path of the file or directory to be uploaded. + """ + path_local_upload = os.path.abspath(path_local_upload) + if os.path.isdir(path_local_upload): + prefix = os.path.join(prefix, os.path.basename(path_local_upload)) + + # Check if bucket already exists, if not create it. + found = self.bucket_exists(bucket_name) + if not found: + self.make_bucket(bucket_name) + else: + print(f"Bucket '{bucket_name}' already exists") + + if os.path.isdir(path_local_upload): + files = get_all_file_paths(path_local_upload) + for file, file_ in tqdm(zip(files[0], files[1]), total=len(files[0]), desc="Uploading", unit="file"): + path_drive = os.path.join(prefix, file_).replace("\\", "/") + self.fput_object( + bucket_name, path_drive, file, + ) + else: + path_local_upload = os.path.abspath(path_local_upload) + path_drive = os.path.join(prefix, os.path.basename(path_local_upload)).replace("\\", "/") + self.fput_object( + bucket_name, path_drive, path_local_upload, + ) + + def download(self,bucket_name: str,prefix:str, dir_local_storage:str): + """Download a file or directory from a Minio bucket. + + Args: + bucket_name (str): The name of the Minio bucket. + prefix (str): The prefix to be added to the file(s) in the bucket. + dir_local_storage (str): The directory to be downloaded. + """ + for obj in self.list_objects(bucket_name, prefix=prefix, recursive=True): + self.fget_object(bucket_name, obj.object_name, os.path.join(dir_local_storage, obj.object_name),progress=Progress()) + +if __name__ == "__main__": + import os + endpoint = os.environ["MINIO_ENDPOINT"] + access_key = os.environ["MINIO_ACCESS_KEY"] + secret_key = os.environ["MINIO_SECRET_KEY"] + client = MinioClient(endpoint, access_key, secret_key) + + bucket_name = "fine-tuned-model" + prefix = "test/test/test/corrector_for_measure_accuracy/google-mt5-base-corrector-8-10(4)+full_testset_answer.csv" + path_local = "./" + client.download(bucket_name, prefix, path_local) \ No newline at end of file diff --git a/py_utility/progress.py b/py_utility/progress.py new file mode 100644 index 0000000..ab35a48 --- /dev/null +++ b/py_utility/progress.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, +# (C) 2018 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module implements a progress printer while communicating with MinIO server + +:copyright: (c) 2018 by MinIO, Inc. +:license: Apache 2.0, see LICENSE for more details. + +""" + +import sys +import time +from queue import Empty, Queue +from threading import Thread + +_BAR_SIZE = 20 +_KILOBYTE = 1024 +_FINISHED_BAR = '#' +_REMAINING_BAR = '-' + +_UNKNOWN_SIZE = '?' +_STR_MEGABYTE = ' MB' + +_HOURS_OF_ELAPSED = '%d:%02d:%02d' +_MINUTES_OF_ELAPSED = '%02d:%02d' + +_RATE_FORMAT = '%5.2f' +_PERCENTAGE_FORMAT = '%3d%%' +_HUMANINZED_FORMAT = '%0.2f' + +_DISPLAY_FORMAT = '|%s| %s/%s %s [elapsed: %s left: %s, %s MB/sec]' + +_REFRESH_CHAR = '\r' + + +class Progress(Thread): + """ + Constructs a :class:`Progress` object. + :param interval: Sets the time interval to be displayed on the screen. + :param stdout: Sets the standard output + + :return: :class:`Progress` object + """ + + def __init__(self, interval=1, stdout=sys.stdout): + Thread.__init__(self) + self.daemon = True + self.total_length = 0 + self.interval = interval + self.object_name = None + + self.last_printed_len = 0 + self.current_size = 0 + + self.display_queue = Queue() + self.initial_time = time.time() + self.stdout = stdout + self.start() + + def set_meta(self, total_length, object_name): + """ + Metadata settings for the object. This method called before uploading + object + :param total_length: Total length of object. + :param object_name: Object name to be showed. + """ + self.total_length = total_length + self.object_name = object_name + self.prefix = self.object_name + ': ' if self.object_name else '' + + def run(self): + displayed_time = 0 + while True: + try: + # display every interval secs + task = self.display_queue.get(timeout=self.interval) + except Empty: + elapsed_time = time.time() - self.initial_time + if elapsed_time > displayed_time: + displayed_time = elapsed_time + self.print_status(current_size=self.current_size, + total_length=self.total_length, + displayed_time=displayed_time, + prefix=self.prefix) + continue + + current_size, total_length = task + displayed_time = time.time() - self.initial_time + self.print_status(current_size=current_size, + total_length=total_length, + displayed_time=displayed_time, + prefix=self.prefix) + self.display_queue.task_done() + if current_size == total_length: + # once we have done uploading everything return + self.done_progress() + return + + def update(self, size): + """ + Update object size to be showed. This method called while uploading + :param size: Object size to be showed. The object size should be in + bytes. + """ + if not isinstance(size, int): + raise ValueError('{} type can not be displayed. ' + 'Please change it to Int.'.format(type(size))) + + self.current_size += size + self.display_queue.put((self.current_size, self.total_length)) + + def done_progress(self): + self.total_length = 0 + self.object_name = None + self.last_printed_len = 0 + self.current_size = 0 + + def print_status(self, current_size, total_length, displayed_time, prefix): + formatted_str = prefix + format_string( + current_size, total_length, displayed_time) + self.stdout.write(_REFRESH_CHAR + formatted_str + ' ' * + max(self.last_printed_len - len(formatted_str), 0)) + self.stdout.flush() + self.last_printed_len = len(formatted_str) + + +def seconds_to_time(seconds): + """ + Consistent time format to be displayed on the elapsed time in screen. + :param seconds: seconds + """ + minutes, seconds = divmod(int(seconds), 60) + hours, m = divmod(minutes, 60) + if hours: + return _HOURS_OF_ELAPSED % (hours, m, seconds) + else: + return _MINUTES_OF_ELAPSED % (m, seconds) + + +def format_string(current_size, total_length, elapsed_time): + """ + Consistent format to be displayed on the screen. + :param current_size: Number of finished object size + :param total_length: Total object size + :param elapsed_time: number of seconds passed since start + """ + + n_to_mb = current_size / _KILOBYTE / _KILOBYTE + elapsed_str = seconds_to_time(elapsed_time) + + rate = _RATE_FORMAT % ( + n_to_mb / elapsed_time) if elapsed_time else _UNKNOWN_SIZE + frac = float(current_size) / total_length + bar_length = int(frac * _BAR_SIZE) + bar = (_FINISHED_BAR * bar_length + + _REMAINING_BAR * (_BAR_SIZE - bar_length)) + percentage = _PERCENTAGE_FORMAT % (frac * 100) + left_str = ( + seconds_to_time( + elapsed_time / current_size * (total_length - current_size)) + if current_size else _UNKNOWN_SIZE) + + humanized_total = _HUMANINZED_FORMAT % ( + total_length / _KILOBYTE / _KILOBYTE) + _STR_MEGABYTE + humanized_n = _HUMANINZED_FORMAT % n_to_mb + _STR_MEGABYTE + + return _DISPLAY_FORMAT % (bar, humanized_n, humanized_total, percentage, + elapsed_str, left_str, rate) From 24caefdf79c6f753214f2be920ccb7e6f207de57 Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 19 Sep 2023 13:29:58 +0700 Subject: [PATCH 2/8] add minio utility and refactor code --- README.md | 8 ++ py_utility/__init__.py | 11 ++- py_utility/file.py | 82 ---------------- py_utility/file_wrapper.py | 89 ++++++++++++++++++ py_utility/log.py | 40 -------- py_utility/log_wropper.py | 87 +++++++++++++++++ py_utility/minio_.py | 78 ---------------- py_utility/minio_wrapper.py | 167 +++++++++++++++++++++++++++++++++ py_utility/path.py | 68 -------------- py_utility/path_wrapper.py | 78 ++++++++++++++++ py_utility/progress.py | 182 ------------------------------------ setup.py | 30 +++++- tests/__init__.py | 0 tests/test_file_wrapper.py | 50 ++++++++++ tests/test_log_wrapper.py | 49 ++++++++++ tests/test_path_wrapper.py | 50 ++++++++++ 16 files changed, 613 insertions(+), 456 deletions(-) delete mode 100644 py_utility/file.py create mode 100644 py_utility/file_wrapper.py delete mode 100644 py_utility/log.py create mode 100644 py_utility/log_wropper.py delete mode 100644 py_utility/minio_.py create mode 100644 py_utility/minio_wrapper.py delete mode 100644 py_utility/path.py create mode 100644 py_utility/path_wrapper.py delete mode 100644 py_utility/progress.py create mode 100644 tests/__init__.py create mode 100644 tests/test_file_wrapper.py create mode 100644 tests/test_log_wrapper.py create mode 100644 tests/test_path_wrapper.py diff --git a/README.md b/README.md index ff61e31..ca18cc7 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,14 @@ You can install py_utility using pip: pip install py_utility ``` +## Testing + +You can run the unit tests using pytest: + +```bash +python -m unittest discover tests +``` + ## Contributing If you find a bug or have an idea for a new feature, please open an issue on the GitHub repository. Pull requests are welcome! \ No newline at end of file diff --git a/py_utility/__init__.py b/py_utility/__init__.py index 51b5161..1b4b13f 100644 --- a/py_utility/__init__.py +++ b/py_utility/__init__.py @@ -1,3 +1,8 @@ -from .file import * -from .log import * -from .path import * \ No newline at end of file +import os + +files = os.listdir(__path__[0]) +modules = ( + x.replace(".py", "") for x in files if x.endswith(".py") and not x.startswith("__") +) +for module in modules: + __import__("py_utility." + module) \ No newline at end of file diff --git a/py_utility/file.py b/py_utility/file.py deleted file mode 100644 index 86883fe..0000000 --- a/py_utility/file.py +++ /dev/null @@ -1,82 +0,0 @@ -import codecs -import json -import pathlib -from typing import Generator, Union - - -def read_file_to_text(path: str, encoding: str = "utf-8", stream_line: bool = False) -> Union[str, Generator[str, None, None]]: - """ - Reads the contents of a file and returns either the entire text or a generator of lines. - - Args: - path (str): The path to the file. - encoding (str, optional): The encoding of the file. Defaults to "utf-8". - stream_line (bool, optional): If True, returns a generator of lines. Defaults to False. - - Returns: - Union[str, Generator[str, None, None]]: Either the entire text or a generator of lines. - """ - if stream_line: - with open(path, "r", encoding=encoding) as f: - for line in f: - yield line.rstrip('\r\n') - else: - with open(path, "r", encoding=encoding) as f: - text = f.read() - return text - - -def write_text(path_file: str, content: str, encoding="utf-8"): - """Write text to file - - Args: - path_file (str): path to file output - content (str): content to write - encoding (str, optional): encoding of file. Defaults to "utf-8". - """ - with codecs.open(path_file, "w", encoding=encoding) as f: - f.write(content) - - -def read_file_config(path_config: str) -> dict: - """Read file config format json - Args: - path_config (str): path to file config - - Returns: - dict: config in dict - """ - with codecs.open(path_config, "r", encoding="utf-8") as f: - config = f.read() - preprocess_config = json.loads(config) - return preprocess_config - - -def get_file_name_without_extension(path_file: str) -> str: - """Get file name without extension - - Args: - path_file (str): path to file - - Returns: - _type_: file name without extension - """ - return pathlib.Path(path_file).stem - - -def get_file_name(path_file: str, tail: str = "", set_extension=None, without_extension=False) -> str: - """ get file name with tail and extension - - Args: - path_file (str): path to file - tail (str, optional): tail of file name. Defaults empty. - without_extension (bool, optional): if True, return file name without extension. Defaults to False. - set_extension (str, optional): set extension of file name. Defaults to None. - Returns: - str: file name with tail and extension - """ - if without_extension: - return pathlib.Path(path_file).stem+"%s" % (tail) - if set_extension == None: - set_extension = pathlib.Path(path_file).suffix - return pathlib.Path(path_file).stem+"%s%s" % (tail, set_extension) diff --git a/py_utility/file_wrapper.py b/py_utility/file_wrapper.py new file mode 100644 index 0000000..93066a1 --- /dev/null +++ b/py_utility/file_wrapper.py @@ -0,0 +1,89 @@ +import json +from pathlib import Path +from typing import Generator + +def read_file_all_text(path: str, encoding: str = "utf-8") -> str: + """ + Reads the entire content of a file and returns it as a single string. + + Args: + path (str): The path to the file. + encoding (str, optional): The encoding of the file. Defaults to "utf-8". + + Returns: + str: The entire content of the file. + """ + with open(path, "r", encoding=encoding) as f: + return f.read() + +def stream_file_by_line(path: str, encoding: str = "utf-8") -> Generator[str, None, None]: + """ + Streams the content of a file line by line. + + Args: + path (str): The path to the file. + encoding (str, optional): The encoding of the file. Defaults to "utf-8". + + Yields: + Generator[str, None, None]: Each line from the file. + """ + with open(path, "r", encoding=encoding) as f: + for line in f: + yield line.rstrip('\r\n') + + +def write_text_to_file(path: str, content: str, encoding: str = "utf-8") -> None: + """Write text to file + + Args: + path (str): Path to the output file. + content (str): Content to write. + encoding (str, optional): Encoding of the file. Defaults to "utf-8". + """ + with open(path, "w", encoding=encoding) as f: + f.write(content) + + +def read_json_config(path: str) -> dict: + """Read JSON config file + + Args: + path (str): Path to the config file. + + Returns: + dict: Configuration as a dictionary. + """ + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def get_file_name_without_extension(path: str) -> str: + """Get file name without extension + + Args: + path (str): Path to the file. + + Returns: + str: File name without extension. + """ + return Path(path).stem + + +def get_file_name(path: str, tail: str = "", set_extension: str = None, without_extension: bool = False) -> str: + """Get file name with optional modifications + + Args: + path (str): Path to the file. + tail (str, optional): Additional tail for the file name. Defaults to "". + set_extension (str, optional): Desired file extension. If None, uses original extension. Defaults to None. + without_extension (bool, optional): If True, returns file name without extension. Defaults to False. + + Returns: + str: Modified file name. + """ + file_name = Path(path).stem + if without_extension: + return file_name + tail + if not set_extension: + set_extension = Path(path).suffix + return file_name + tail + set_extension diff --git a/py_utility/log.py b/py_utility/log.py deleted file mode 100644 index e87fe29..0000000 --- a/py_utility/log.py +++ /dev/null @@ -1,40 +0,0 @@ -import json -import logging - -class JSONFormatter(logging.Formatter): - def format(self, record): - log_data = { - 'timestamp': record.created, - 'level': record.levelname, - 'message': record.getMessage(), - 'module': record.module, - 'line': record.lineno, - 'funcName': record.funcName, - 'pathname': record.pathname - } - return json.dumps(log_data, ensure_ascii=False) - -def setup_logging(log_file_path: str ,mode='a', encoding='utf-8', log_level=logging.DEBUG) -> logging.Logger: - """ - Sets up a logger with a JSON formatter and a file handler. - - Args: - log_file_path (str): path to the log file. - mode (str, optional): file mode. Defaults to 'a'. - encoding (str, optional): file encoding. Defaults to 'utf-8'. - log_level (int, optional): logging level. Defaults to logging.DEBUG. - - Returns: - logging.Logger: configured logger object. - """ - logger = logging.getLogger() - logger.setLevel(log_level) - - formatter = JSONFormatter() - - file_handler = logging.FileHandler(log_file_path, mode=mode, encoding=encoding) - file_handler.setFormatter(formatter) - - logger.addHandler(file_handler) - - return logger \ No newline at end of file diff --git a/py_utility/log_wropper.py b/py_utility/log_wropper.py new file mode 100644 index 0000000..ba2a899 --- /dev/null +++ b/py_utility/log_wropper.py @@ -0,0 +1,87 @@ +import logging +from typing import Optional, Union +from pathlib import Path +import json + +class JSONFormatter(logging.Formatter): + """ + A custom formatter for logging messages as JSON. + + Methods: + format: Returns the log message formatted as a JSON string. + """ + + def format(self, record: logging.LogRecord) -> str: + """Return the log message formatted as JSON.""" + log_data = { + 'timestamp': record.created, + 'level': record.levelname, + 'message': record.getMessage(), + 'module': record.module, + 'line': record.lineno, + 'funcName': record.funcName, + 'pathname': record.pathname + } + return json.dumps(log_data, ensure_ascii=False) + +class LoggerSetup: + """ + A class to configure a logger with options for JSON formatted logging to file and console. + + Methods: + add_file_handler: Adds a file handler to the logger. + add_console_handler: Adds a console handler to the logger. + set_custom_format: Sets a custom formatter for logging messages. + get_logger: Returns the configured logger. + """ + + def __init__(self, log_level: int = logging.DEBUG): + """Initialize the LoggerSetup with the desired log level. + + Examples: + >>> logger_setup = LoggerSetup(log_level=logging.DEBUG) + >>> custom_format = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + >>> logger_setup.set_custom_format(custom_format) + >>> logger_setup.add_file_handler("logfile.log") + >>> logger_setup.add_console_handler() + >>> logger = logger_setup.get_logger() + >>> logger.debug("This is a debug message.") + """ + self.logger = logging.getLogger() + self.logger.setLevel(log_level) + self.formatter = JSONFormatter() + + def add_file_handler(self, log_file_path: Optional[Path] = None, mode: str = 'a', encoding: str = 'utf-8') -> None: + """ + Add a file handler to the logger. + + Args: + log_file_path (Path): The path to the log file. Defaults to None. + mode (str): File mode. Defaults to 'a'. + encoding (str): File encoding. Defaults to 'utf-8'. + """ + if log_file_path: + file_handler = logging.FileHandler(log_file_path, mode=mode, encoding=encoding) + file_handler.setFormatter(self.formatter) + self.logger.addHandler(file_handler) + + def add_console_handler(self) -> None: + """Add a console handler to the logger.""" + console_handler = logging.StreamHandler() + console_handler.setFormatter(self.formatter) + self.logger.addHandler(console_handler) + + def set_custom_format(self, fmt: Union[logging.Formatter, None]) -> None: + """ + Set a custom formatter for logging messages. + + Args: + fmt (logging.Formatter | None): The desired logging formatter. + """ + for handler in self.logger.handlers: + handler.setFormatter(fmt) + self.formatter = fmt + + def get_logger(self) -> logging.Logger: + """Return the configured logger.""" + return self.logger diff --git a/py_utility/minio_.py b/py_utility/minio_.py deleted file mode 100644 index 72a1261..0000000 --- a/py_utility/minio_.py +++ /dev/null @@ -1,78 +0,0 @@ -from typing import List -from minio import Minio -from minio.error import S3Error -import os -from tqdm import tqdm -from py_utility.progress import Progress - -def get_all_file_paths(directory): - directory = os.path.abspath(directory) - file_paths = [] - - for root, directories, files in os.walk(directory): - for filename in files: - filepath = os.path.join(root, filename) - file_paths.append(filepath) - file_paths_ = [file[len(directory)+1:] for file in file_paths] - return file_paths, file_paths_ - -class MinioClient(Minio): - def __init__(self, endpoint, access_key=None, secret_key=None, session_token=None, secure=True, region=None, http_client=None, credentials=None, cert_check=True): - super().__init__(endpoint, access_key, secret_key, session_token, secure, region, http_client, credentials, cert_check) - - def upload(self,bucket_name: str,prefix:str, path_local_upload:str): - """ - Uploads a file or directory to a Minio bucket. - - Args: - bucket_name (str): The name of the Minio bucket. - prefix (str): The prefix to be added to the file(s) in the bucket. - path_local_upload (str): The path of the file or directory to be uploaded. - """ - path_local_upload = os.path.abspath(path_local_upload) - if os.path.isdir(path_local_upload): - prefix = os.path.join(prefix, os.path.basename(path_local_upload)) - - # Check if bucket already exists, if not create it. - found = self.bucket_exists(bucket_name) - if not found: - self.make_bucket(bucket_name) - else: - print(f"Bucket '{bucket_name}' already exists") - - if os.path.isdir(path_local_upload): - files = get_all_file_paths(path_local_upload) - for file, file_ in tqdm(zip(files[0], files[1]), total=len(files[0]), desc="Uploading", unit="file"): - path_drive = os.path.join(prefix, file_).replace("\\", "/") - self.fput_object( - bucket_name, path_drive, file, - ) - else: - path_local_upload = os.path.abspath(path_local_upload) - path_drive = os.path.join(prefix, os.path.basename(path_local_upload)).replace("\\", "/") - self.fput_object( - bucket_name, path_drive, path_local_upload, - ) - - def download(self,bucket_name: str,prefix:str, dir_local_storage:str): - """Download a file or directory from a Minio bucket. - - Args: - bucket_name (str): The name of the Minio bucket. - prefix (str): The prefix to be added to the file(s) in the bucket. - dir_local_storage (str): The directory to be downloaded. - """ - for obj in self.list_objects(bucket_name, prefix=prefix, recursive=True): - self.fget_object(bucket_name, obj.object_name, os.path.join(dir_local_storage, obj.object_name),progress=Progress()) - -if __name__ == "__main__": - import os - endpoint = os.environ["MINIO_ENDPOINT"] - access_key = os.environ["MINIO_ACCESS_KEY"] - secret_key = os.environ["MINIO_SECRET_KEY"] - client = MinioClient(endpoint, access_key, secret_key) - - bucket_name = "fine-tuned-model" - prefix = "test/test/test/corrector_for_measure_accuracy/google-mt5-base-corrector-8-10(4)+full_testset_answer.csv" - path_local = "./" - client.download(bucket_name, prefix, path_local) \ No newline at end of file diff --git a/py_utility/minio_wrapper.py b/py_utility/minio_wrapper.py new file mode 100644 index 0000000..694c78c --- /dev/null +++ b/py_utility/minio_wrapper.py @@ -0,0 +1,167 @@ +from minio import Minio +from tqdm import tqdm +import os +from multiprocessing import Pool, cpu_count +from typing import List, Tuple +import logging +from concurrent.futures import ThreadPoolExecutor + +class MinioWrapper: + """ + Wrapper class for MinIO operations. + + Attributes: + - minio_client (Minio): The Minio client object. + - endpoint (str): Minio server endpoint. + - access_key (str): Access key for Minio server. + - secret_key (str): Secret key for Minio server. + """ + def __init__(self, endpoint: str, access_key: str = None, secret_key: str = None, secure: bool = True): + self.minio_client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure) + logging.info(f"Minio client created for endpoint: {endpoint}") + self.endpoint = endpoint + self.access_key = access_key + self.secret_key = secret_key + + @staticmethod + def get_all_file_paths(directory: str) -> Tuple[List[str], List[str]]: + """ + Retrieves the absolute paths and relative paths of all files in the given directory. + + Parameters: + - directory (str): The directory whose files' paths are to be retrieved. + + Returns: + - Tuple[List[str], List[str]]: A tuple containing two lists: + 1. A list of absolute file paths. + 2. A list of relative file paths from the input directory. + """ + directory = os.path.abspath(directory) + abspath_files = [os.path.join(root, filename) + for root, _, files in os.walk(directory) for filename in files] + relative_paths = [file[len(directory)+1:] for file in abspath_files] + return abspath_files, relative_paths + + @staticmethod + def upload_file(args: Tuple[str, str, str, str, str, str]) -> str: + """ + Uploads a single file to a MinIO bucket. + + Parameters: + - args (Tuple[str, str, str, str, str, str]): A tuple containing the following: + 1. Minio server endpoint. + 2. Access key for Minio server. + 3. Secret key for Minio server. + 4. Target Minio bucket name. + 5. Local path of the file to be uploaded. + 6. Remote path (including filename) where the file will be stored in the bucket. + + Returns: + - str: A string indicating the success status ("Success") or the error message. + """ + minio_endpoint, minio_access_key, minio_secret_key, bucket_name, local_path, remote_path = args + minio_client = Minio(minio_endpoint, minio_access_key, minio_secret_key) + try: + minio_client.fput_object(bucket_name, remote_path, local_path) + return "Success" + except Exception as err: + return f"Upload Error for {local_path} : {err}" + + def upload(self, bucket_name: str, path_local_upload: str, prefix: str = "") -> None: + """ + Uploads files or directories to a specified MinIO bucket. + + If the provided path represents a directory, all files within it are uploaded with their + relative paths maintained in the bucket. If the path represents a single file, only that file + is uploaded. The upload leverages multiprocessing for enhanced speed. + + Parameters: + - bucket_name (str): The target Minio bucket where the files/directories will be uploaded. + - path_local_upload (str): The local path of the file or directory to be uploaded. + - prefix (str, optional): The prefix or folder name within the bucket where the files will be uploaded. Defaults to "". + + Returns: + - None: Files are uploaded to the MinIO bucket and no explicit return value is provided. + + Raises: + - Exceptions related to file upload will be logged. + + Example: + >>> client = MinioWrapper(endpoint="localhost:9000", access_key="YOUR_ACCESS_KEY", secret_key="YOUR_SECRET_KEY") + >>> client.upload(bucket_name="mybucket", path_local_upload="/path/to/local/data", prefix="remote/folder/") + """ + path_local_upload = os.path.abspath(path_local_upload) + upload_args = [] + + if os.path.isdir(path_local_upload): + prefix = os.path.join(prefix, os.path.basename(path_local_upload)) + files = MinioWrapper.get_all_file_paths(path_local_upload) + upload_args = [(self.endpoint, self.access_key, self.secret_key, bucket_name, local_file, os.path.join(prefix, remote_file).replace("\\", "/")) + for local_file, remote_file in zip(files[0], files[1])] + else: + remote_path = os.path.join(prefix, os.path.basename(path_local_upload)).replace("\\", "/") + upload_args.append((self.endpoint, self.access_key, self.secret_key, bucket_name, path_local_upload, remote_path)) + + # Use multiprocessing for the uploads + with Pool(processes=cpu_count()) as pool: + results = list(tqdm(pool.imap_unordered(MinioWrapper.upload_file, upload_args), total=len(upload_args), desc="Files Uploaded", unit="file")) + + # Handle and display errors + errors = [result for result in results if result != "Success"] + for error in errors: + logging.error(error) + + + def download_files(self, bucket_name, prefix="", recursive=False, destination_path="", max_workers=10): + """ + Download all files from the specified bucket with optional prefix and recursion. + + Args: + - bucket_name (str): Name of the bucket in minio. + - prefix (str, optional): Prefix or folder name within the bucket. Defaults to "". + - recursive (bool, optional): Whether or not to download files recursively. Defaults to False. + - destination_path (str, optional): Local directory where the files will be downloaded to. Defaults to the current directory. + - max_workers (int, optional): Maximum number of threads to use for concurrent downloads. Defaults to 10. + """ + + os.makedirs(destination_path, exist_ok=True) + objects_to_download = [obj.object_name for obj in self.minio_client.list_objects(bucket_name, prefix=prefix, recursive=recursive)] + + def _download_to_dest(bucket_name, object_name, destination, pbar=None): + dest_file_path = os.path.join(destination, object_name) + os.makedirs(os.path.dirname(dest_file_path), exist_ok=True) + self.minio_client.fget_object(bucket_name, object_name, dest_file_path) + if pbar: + pbar.update(1) + + with tqdm(total=len(objects_to_download), desc="Downloading files", unit="file") as pbar: + with ThreadPoolExecutor(max_workers=max_workers) as executor: + executor.map(_download_to_dest, [bucket_name]*len(objects_to_download), objects_to_download, [destination_path]*len(objects_to_download), [pbar]*len(objects_to_download)) + + def download_file(self, bucket_name: str, file_name: str, file_output: str = None) -> None: + """ + Downloads a specific file from the given MinIO bucket. + + Args: + - bucket_name (str): The name of the bucket in MinIO from which the file needs to be downloaded. + - file_name (str): The name (or path) of the file within the bucket to download. + - file_output (str, optional): The desired local name (or path) for the downloaded file. + If not provided, the file will be saved with its original name from the bucket. + + Returns: + - None: The function saves the downloaded file to the local filesystem and does not return any value. + + Raises: + - S3Error: If there is an issue related to the S3 operation, e.g., a file or bucket does not exist. + - ResponseError: If there is a network-related error during the call. + + Example: + >>> client = MinioClient(endpoint="localhost:9000", access_key="YOUR_ACCESS_KEY", secret_key="YOUR_SECRET_KEY") + >>> client.download_file(bucket_name="mybucket", file_name="data.txt", file_output="local_data.txt") + """ + + + file_output_name = file_name + if file_output is not None: + file_output_name = file_output + self.minio_client.fget_object(bucket_name, file_name, file_output_name) \ No newline at end of file diff --git a/py_utility/path.py b/py_utility/path.py deleted file mode 100644 index f992e3b..0000000 --- a/py_utility/path.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -from typing import List - -def get_all_files(path_input: str, endswith: str = None) -> List[str]: - """Get all files in your path input - - Args: - path_input (str): path input - endswith (str, optional): endswith file. Defaults to None. if None, return all file. - - Returns: - List[str]: list of path file - """ - if os.path.isfile(path_input): - return [path_input] - - file_list = [] - for root, dirs, files in os.walk(path_input): - for file in files: - if endswith is None or file.endswith(endswith): - file_list.append(os.path.join(root, file)) - - return file_list - -def get_current_folder_name(path_file: str)-> str: - """Get current folder name - - Args: - path_file (str): path file - - Returns: - _type_: current folder name - """ - if os.path.isfile(path_file): - return os.path.basename(os.path.dirname(path_file)) - return os.path.basename((os.path.abspath(path_file))) - -def get_previous_path(path_input: str, previous: int = 1) -> str: - """ - Returns the path `previous` directories above `path_input`. - - Args: - path_input (str): The starting path. - previous (int, optional): The number of directories to go up. Defaults to 1. - - Returns: - str: The resulting path. - """ - if previous < 1: - return os.path.abspath(path_input) - - path_input = os.path.abspath(path_input) - for i in range(previous): - path_input = os.path.dirname(path_input) - return path_input - -def get_all_directory(path_input)->str: - """Get all directory in your path input - - Args: - path_input (_type_): path input - - Returns: - str: list of path is directory - """ - path_input = os.path.abspath(path_input) - return [os.path.join(path_input, name) for name in os.listdir(path_input) - if os.path.isdir(os.path.join(path_input, name))] \ No newline at end of file diff --git a/py_utility/path_wrapper.py b/py_utility/path_wrapper.py new file mode 100644 index 0000000..6206e8c --- /dev/null +++ b/py_utility/path_wrapper.py @@ -0,0 +1,78 @@ +import os +from typing import List + +def get_all_files(path_input: str, endswith: str = None, recursive: bool = True) -> List[str]: + """ + Get all files from the specified path. + + Args: + - path_input (str): Path to start the search from. + - endswith (str, optional): File extension filter. If None, returns all files. + - recursive (bool, optional): If True, search for files recursively. Defaults to True. + + Returns: + - List[str]: List of file paths. + """ + if os.path.isfile(path_input): + return [path_input] + + if recursive: + files = [ + os.path.join(root, file) + for root, _, files in os.walk(path_input) + for file in files + if endswith is None or file.endswith(endswith) + ] + else: + files = [ + os.path.join(path_input, file) + for file in os.listdir(path_input) + if os.path.isfile(os.path.join(path_input, file)) and (endswith is None or file.endswith(endswith)) + ] + + return files + +def get_current_folder_name(path: str) -> str: + """ + Get the name of the folder containing the specified path. + + Args: + - path (str): Path to a file or folder. + + Returns: + - str: Name of the containing folder. + """ + return os.path.basename(os.path.dirname(os.path.abspath(path))) + +def get_previous_path(path_input: str, previous: int = 1) -> str: + """ + Returns the path `previous` directories above `path_input`. + + Args: + - path_input (str): The starting path. + - previous (int, optional): The number of directories to go up. Defaults to 1. + + Returns: + - str: The resulting path. + """ + path = os.path.abspath(path_input) + for _ in range(previous): + path = os.path.dirname(path) + return path + +def get_all_directories(path_input: str) -> List[str]: + """ + Get all directories from the specified path. + + Args: + - path_input (str): Path to start the search from. + + Returns: + - List[str]: List of directory paths. + """ + path_input = os.path.abspath(path_input) + return [ + os.path.join(path_input, name) + for name in os.listdir(path_input) + if os.path.isdir(os.path.join(path_input, name)) + ] diff --git a/py_utility/progress.py b/py_utility/progress.py deleted file mode 100644 index ab35a48..0000000 --- a/py_utility/progress.py +++ /dev/null @@ -1,182 +0,0 @@ -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, -# (C) 2018 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This module implements a progress printer while communicating with MinIO server - -:copyright: (c) 2018 by MinIO, Inc. -:license: Apache 2.0, see LICENSE for more details. - -""" - -import sys -import time -from queue import Empty, Queue -from threading import Thread - -_BAR_SIZE = 20 -_KILOBYTE = 1024 -_FINISHED_BAR = '#' -_REMAINING_BAR = '-' - -_UNKNOWN_SIZE = '?' -_STR_MEGABYTE = ' MB' - -_HOURS_OF_ELAPSED = '%d:%02d:%02d' -_MINUTES_OF_ELAPSED = '%02d:%02d' - -_RATE_FORMAT = '%5.2f' -_PERCENTAGE_FORMAT = '%3d%%' -_HUMANINZED_FORMAT = '%0.2f' - -_DISPLAY_FORMAT = '|%s| %s/%s %s [elapsed: %s left: %s, %s MB/sec]' - -_REFRESH_CHAR = '\r' - - -class Progress(Thread): - """ - Constructs a :class:`Progress` object. - :param interval: Sets the time interval to be displayed on the screen. - :param stdout: Sets the standard output - - :return: :class:`Progress` object - """ - - def __init__(self, interval=1, stdout=sys.stdout): - Thread.__init__(self) - self.daemon = True - self.total_length = 0 - self.interval = interval - self.object_name = None - - self.last_printed_len = 0 - self.current_size = 0 - - self.display_queue = Queue() - self.initial_time = time.time() - self.stdout = stdout - self.start() - - def set_meta(self, total_length, object_name): - """ - Metadata settings for the object. This method called before uploading - object - :param total_length: Total length of object. - :param object_name: Object name to be showed. - """ - self.total_length = total_length - self.object_name = object_name - self.prefix = self.object_name + ': ' if self.object_name else '' - - def run(self): - displayed_time = 0 - while True: - try: - # display every interval secs - task = self.display_queue.get(timeout=self.interval) - except Empty: - elapsed_time = time.time() - self.initial_time - if elapsed_time > displayed_time: - displayed_time = elapsed_time - self.print_status(current_size=self.current_size, - total_length=self.total_length, - displayed_time=displayed_time, - prefix=self.prefix) - continue - - current_size, total_length = task - displayed_time = time.time() - self.initial_time - self.print_status(current_size=current_size, - total_length=total_length, - displayed_time=displayed_time, - prefix=self.prefix) - self.display_queue.task_done() - if current_size == total_length: - # once we have done uploading everything return - self.done_progress() - return - - def update(self, size): - """ - Update object size to be showed. This method called while uploading - :param size: Object size to be showed. The object size should be in - bytes. - """ - if not isinstance(size, int): - raise ValueError('{} type can not be displayed. ' - 'Please change it to Int.'.format(type(size))) - - self.current_size += size - self.display_queue.put((self.current_size, self.total_length)) - - def done_progress(self): - self.total_length = 0 - self.object_name = None - self.last_printed_len = 0 - self.current_size = 0 - - def print_status(self, current_size, total_length, displayed_time, prefix): - formatted_str = prefix + format_string( - current_size, total_length, displayed_time) - self.stdout.write(_REFRESH_CHAR + formatted_str + ' ' * - max(self.last_printed_len - len(formatted_str), 0)) - self.stdout.flush() - self.last_printed_len = len(formatted_str) - - -def seconds_to_time(seconds): - """ - Consistent time format to be displayed on the elapsed time in screen. - :param seconds: seconds - """ - minutes, seconds = divmod(int(seconds), 60) - hours, m = divmod(minutes, 60) - if hours: - return _HOURS_OF_ELAPSED % (hours, m, seconds) - else: - return _MINUTES_OF_ELAPSED % (m, seconds) - - -def format_string(current_size, total_length, elapsed_time): - """ - Consistent format to be displayed on the screen. - :param current_size: Number of finished object size - :param total_length: Total object size - :param elapsed_time: number of seconds passed since start - """ - - n_to_mb = current_size / _KILOBYTE / _KILOBYTE - elapsed_str = seconds_to_time(elapsed_time) - - rate = _RATE_FORMAT % ( - n_to_mb / elapsed_time) if elapsed_time else _UNKNOWN_SIZE - frac = float(current_size) / total_length - bar_length = int(frac * _BAR_SIZE) - bar = (_FINISHED_BAR * bar_length + - _REMAINING_BAR * (_BAR_SIZE - bar_length)) - percentage = _PERCENTAGE_FORMAT % (frac * 100) - left_str = ( - seconds_to_time( - elapsed_time / current_size * (total_length - current_size)) - if current_size else _UNKNOWN_SIZE) - - humanized_total = _HUMANINZED_FORMAT % ( - total_length / _KILOBYTE / _KILOBYTE) + _STR_MEGABYTE - humanized_n = _HUMANINZED_FORMAT % n_to_mb + _STR_MEGABYTE - - return _DISPLAY_FORMAT % (bar, humanized_n, humanized_total, percentage, - elapsed_str, left_str, rate) diff --git a/setup.py b/setup.py index f0b8f63..d70710b 100644 --- a/setup.py +++ b/setup.py @@ -1,20 +1,44 @@ import setuptools +import subprocess +import os + +cf_remote_version = ( + subprocess.run(["git", "describe", "--tags"], stdout=subprocess.PIPE) + .stdout.decode("utf-8") + .strip() +) + +if "-" in cf_remote_version: + # when not on tag, git describe outputs: "1.3.3-22-gdf81228" + # pip has gotten strict with version numbers + # so change it to: "1.3.3+22.git.gdf81228" + # See: https://peps.python.org/pep-0440/#local-version-segments + v,i,s = cf_remote_version.split("-") + cf_remote_version = v + "+" + i + ".git." + s + +assert "-" not in cf_remote_version +assert "." in cf_remote_version with open("README.md", "r") as f: long_description = f.read() setuptools.setup( name="py_utility", - version="2.0.0", + version=cf_remote_version, author="Tinnawong saelao", author_email="tinnawong2010@hotmail.com", description="python utility for research", long_description=long_description, long_description_content_type="text/markdown", + url="https://github.com/tinnawong/py_utility", packages=setuptools.find_packages(), + package_data={"py_utility": ["VERSION"]}, + include_package_data=True, classifiers=[ "Programming Language :: Python :: 3", "Operating System :: OS Independent", ], - install_requires= [], -) + install_requires= [ + "minio==7.1.15", + ], +) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_file_wrapper.py b/tests/test_file_wrapper.py new file mode 100644 index 0000000..b8bbad4 --- /dev/null +++ b/tests/test_file_wrapper.py @@ -0,0 +1,50 @@ +import unittest +from unittest.mock import mock_open, patch +from py_utility import file_wrapper as fw + +class TestFileFunctions(unittest.TestCase): + + def test_read_file_all_text(self): + m = mock_open(read_data="test content") + with patch("builtins.open", m): + content = fw.read_file_all_text("fakepath.txt") + self.assertEqual(content, "test content") + + def test_stream_file_by_line(self): + m = mock_open(read_data="line1\nline2\nline3") + with patch("builtins.open", m): + lines = list(fw.stream_file_by_line("fakepath.txt")) + self.assertEqual(lines, ["line1", "line2", "line3"]) + + def test_write_text_to_file(self): + m = mock_open() + with patch("builtins.open", m): + fw.write_text_to_file("fakepath.txt", "test content") + m.assert_called_once_with("fakepath.txt", "w", encoding="utf-8") + handle = m() + handle.write.assert_called_once_with("test content") + + def test_read_json_config(self): + mock_json_content = '{"key": "value"}' + m = mock_open(read_data=mock_json_content) + with patch("builtins.open", m), patch("json.load", return_value={"key": "value"}) as mock_json: + result = fw.read_json_config("fakepath.json") + self.assertEqual(result, {"key": "value"}) + mock_json.assert_called_once() + + def test_get_file_name_without_extension(self): + result = fw.get_file_name_without_extension("directory/filename.extension") + self.assertEqual(result, "filename") + + def test_get_file_name(self): + result = fw.get_file_name("directory/filename.extension", tail="_tail", set_extension=".newext") + self.assertEqual(result, "filename_tail.newext") + + result_no_ext = fw.get_file_name("directory/filename.extension", without_extension=True) + self.assertEqual(result_no_ext, "filename") + + result_original_ext = fw.get_file_name("directory/filename.extension", tail="_tail") + self.assertEqual(result_original_ext, "filename_tail.extension") + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_log_wrapper.py b/tests/test_log_wrapper.py new file mode 100644 index 0000000..00499bf --- /dev/null +++ b/tests/test_log_wrapper.py @@ -0,0 +1,49 @@ +import unittest +from unittest.mock import patch, mock_open +import logging +from py_utility import log_wropper as lw + +class TestLoggerSetup(unittest.TestCase): + + def setUp(self): + # Clear existing handlers + logging.getLogger().handlers = [] + + # Initialize a logger setup instance before each test + self.logger_setup = lw.LoggerSetup() + + def test_initialization(self): + self.assertEqual(self.logger_setup.logger.level, logging.DEBUG) + self.assertIsInstance(self.logger_setup.formatter, lw.JSONFormatter) + + @patch('logging.FileHandler._open', mock_open()) + def test_add_file_handler(self): + self.logger_setup.add_file_handler(log_file_path="test.log") + self.assertEqual(len(self.logger_setup.logger.handlers), 1) + self.assertIsInstance(self.logger_setup.logger.handlers[0], logging.FileHandler) + + @patch.object(logging.StreamHandler, 'emit') + def test_add_console_handler(self, mock_emit): + self.logger_setup.add_console_handler() + log_message = "This is a test message." + logger = self.logger_setup.get_logger() + logger.info(log_message) # Make sure to log a message! + + mock_emit.assert_called() + # The emit method takes a LogRecord as argument. So, you should access its `msg` attribute. + self.assertIn(log_message, mock_emit.call_args[0][0].msg) + + def test_set_custom_format(self): + custom_format = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + self.logger_setup.add_console_handler() + self.logger_setup.set_custom_format(custom_format) + + for handler in self.logger_setup.logger.handlers: + self.assertEqual(handler.formatter, custom_format) + + def test_get_logger(self): + logger = self.logger_setup.get_logger() + self.assertEqual(logger, self.logger_setup.logger) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_path_wrapper.py b/tests/test_path_wrapper.py new file mode 100644 index 0000000..907167c --- /dev/null +++ b/tests/test_path_wrapper.py @@ -0,0 +1,50 @@ +import os +import tempfile +import unittest +from py_utility import path_wrapper as pw +class TestFileOperations(unittest.TestCase): + + def setUp(self): + # Creating a temporary directory for testing + self.test_dir = tempfile.TemporaryDirectory() + + # Test file and directory + self.test_file_path = os.path.join(self.test_dir.name, "test.txt") + with open(self.test_file_path, 'w') as f: + f.write("test content") + + self.test_subdir = os.path.join(self.test_dir.name, "subdir") + os.makedirs(self.test_subdir) + + def tearDown(self): + # Cleaning up the temporary directory + self.test_dir.cleanup() + + def test_get_all_files(self): + # Recursive search + files = pw.get_all_files(self.test_dir.name, recursive=True) + self.assertIn(self.test_file_path, files) + + # Non-recursive search + files = pw.get_all_files(self.test_dir.name, recursive=False) + self.assertIn(self.test_file_path, files) + self.assertNotIn(self.test_subdir, files) + + # Filter by extension + files = pw.get_all_files(self.test_dir.name, endswith=".txt", recursive=False) + self.assertIn(self.test_file_path, files) + + def test_get_current_folder_name(self): + folder_name = pw.get_current_folder_name(self.test_file_path) + self.assertEqual(folder_name, os.path.basename(self.test_dir.name)) + + def test_get_previous_path(self): + prev_path = pw.get_previous_path(self.test_file_path) + self.assertEqual(prev_path, self.test_dir.name) + + def test_get_all_directories(self): + dirs = pw.get_all_directories(self.test_dir.name) + self.assertIn(self.test_subdir, dirs) + +if __name__ == "__main__": + unittest.main() From b27fe357f267b10cc579e1747de961f866ff008a Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 19 Sep 2023 16:48:19 +0700 Subject: [PATCH 3/8] delete gitlab ci --- .gitlab-ci.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index c5032ce..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Base image: really weightless -image: python:latest - -stages: - - develop - - production - -develop: - stage: develop - script: python setup.py bdist_wheel - artifacts: - paths: - - dist/ - expire_in: 1 week - only: - - develop - -master: - stage: production - script: - - pip install twine - - python setup.py sdist bdist_wheel - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --verbose --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/* - only: - - master - - /^(v)[0-9]*.[0-9]*.[0-9]*$/ \ No newline at end of file From 8a345fd62f6337b733497a6782b2ec60cb4ffc53 Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 19 Sep 2023 16:48:43 +0700 Subject: [PATCH 4/8] add github workflow --- .github/workflows/python-publish.yml | 33 +++++++++++++++++++ .github/workflows/python-tests.yml | 48 ++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 82 insertions(+) create mode 100644 .github/workflows/python-publish.yml create mode 100644 .github/workflows/python-tests.yml create mode 100644 requirements.txt diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..f34ebb8 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,33 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [published] + push: + tags: [feature-*] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml new file mode 100644 index 0000000..8aafa3c --- /dev/null +++ b/.github/workflows/python-tests.yml @@ -0,0 +1,48 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python tests + +on: + push: + branches: [master, develop] + tags: ['v*.*.*',feature-*] + pull_request: + branches: [master, develop] + +jobs: + test: + + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, 3.9, 3.10] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest setuptools wheel + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest + - name: Install + run: | + python setup.py sdist bdist_wheel + pip install dist/py_utility-*.whl \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1ead088 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +minio==7.1.15 \ No newline at end of file From 5c5bddb2a536c4d7e13f858cff0fdcbeb095b705 Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 19 Sep 2023 17:45:14 +0700 Subject: [PATCH 5/8] fix ci test --- .github/workflows/python-tests.yml | 2 +- requirements.txt | 3 ++- setup.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 8aafa3c..a45507e 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 diff --git a/requirements.txt b/requirements.txt index 1ead088..c5828ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -minio==7.1.15 \ No newline at end of file +minio==7.1.15 +tqdm==4.64.1 \ No newline at end of file diff --git a/setup.py b/setup.py index d70710b..c0dc6bb 100644 --- a/setup.py +++ b/setup.py @@ -40,5 +40,6 @@ ], install_requires= [ "minio==7.1.15", + "tqdm==4.64.1" ], ) \ No newline at end of file From 8bb4927638e3138dc44e24f388bda6408df210bc Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 19 Sep 2023 17:48:49 +0700 Subject: [PATCH 6/8] add test python 3.11 and 3.12 --- .github/workflows/python-tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index a45507e..400e8d3 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -6,19 +6,18 @@ name: Python tests on: push: branches: [master, develop] - tags: ['v*.*.*',feature-*] + tags: ["v*.*.*", feature-*] pull_request: branches: [master, develop] jobs: test: - runs-on: ubuntu-20.04 - + strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, 3.10, 3.11, 3.12] steps: - uses: actions/checkout@v2 @@ -45,4 +44,4 @@ jobs: - name: Install run: | python setup.py sdist bdist_wheel - pip install dist/py_utility-*.whl \ No newline at end of file + pip install dist/py_utility-*.whl From ac8f905686be3623916791024c7e12cf97d510c0 Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 19 Sep 2023 17:53:27 +0700 Subject: [PATCH 7/8] update python version --- .github/workflows/python-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 400e8d3..9961746 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -6,7 +6,6 @@ name: Python tests on: push: branches: [master, develop] - tags: ["v*.*.*", feature-*] pull_request: branches: [master, develop] @@ -17,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, 3.10, 3.11, 3.12] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 From 9e11e2eb76093540ec5f64e159c4197b5534db36 Mon Sep 17 00:00:00 2001 From: tinnawong Date: Tue, 19 Sep 2023 18:19:32 +0700 Subject: [PATCH 8/8] remove test python 3.12 --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 9961746..492a41f 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2