From a416947959b1cdf2728454d6cdd260d3f33900ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Gruszczy=C5=84ski?= Date: Fri, 3 Nov 2023 12:02:51 +0100 Subject: [PATCH] fix: replace src module with package name module --- .gitignore | 162 ++++++++++++++++++++++++ __init__.py | 2 +- examples/unimorph_test_example.py | 36 ++++-- {src => unimorph_inflect}/__init__.py | 0 {src => unimorph_inflect}/inflection.py | 0 {src => unimorph_inflect}/myutil.py | 0 utils/resources.py | 155 ++++++++++++++++++----- 7 files changed, 308 insertions(+), 47 deletions(-) create mode 100644 .gitignore rename {src => unimorph_inflect}/__init__.py (100%) rename {src => unimorph_inflect}/inflection.py (100%) rename {src => unimorph_inflect}/myutil.py (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dbce267 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +.DS_Store diff --git a/__init__.py b/__init__.py index 1afaace..ea7a508 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,4 @@ from unimorph_inflect.utils.resources import download from unimorph_inflect.utils.resources import languages as supported_languages from unimorph_inflect._version import __version__ -from unimorph_inflect.src.inflection import inflect +from unimorph_inflect.unimorph_inflect.inflection import inflect diff --git a/examples/unimorph_test_example.py b/examples/unimorph_test_example.py index b6230cf..9f13626 100644 --- a/examples/unimorph_test_example.py +++ b/examples/unimorph_test_example.py @@ -5,28 +5,41 @@ import argparse import os -#from unimorph_inflect import download +# from unimorph_inflect import download from unimorph_inflect.utils.resources import download, DEFAULT_MODEL_DIR from unimorph_inflect import inflect -from unimorph_inflect.src.myutil import simple_read_data +from unimorph_inflect.unimorph_inflect.myutil import simple_read_data -if __name__ == '__main__': +if __name__ == "__main__": # get arguments parser = argparse.ArgumentParser() # main arguments - parser.add_argument('-d', '--models-dir', help='location of models files | default: ~/unimorph_inflect_resources', - default=DEFAULT_MODEL_DIR) - parser.add_argument('-l', '--language', help='language of text | default: en', default='eng') - parser.add_argument('-t', '--testfile', help='file to test on (in Unimorph format) | default: ', default='') - parser.add_argument('-o', '--output', help='output file path', default=None) + parser.add_argument( + "-d", + "--models-dir", + help="location of models files | default: ~/unimorph_inflect_resources", + default=DEFAULT_MODEL_DIR, + ) + parser.add_argument( + "-l", "--language", help="language of text | default: en", default="eng" + ) + parser.add_argument( + "-t", + "--testfile", + help="file to test on (in Unimorph format) | default: ", + default="", + ) + parser.add_argument("-o", "--output", help="output file path", default=None) # misc arguments - parser.add_argument('--force-download', help='force download of models', action='store_true') + parser.add_argument( + "--force-download", help="force download of models", action="store_true" + ) args = parser.parse_args() # set output file path if args.output is None: - output_file_path = 'text_file.out' + output_file_path = "text_file.out" else: output_file_path = args.output @@ -35,8 +48,7 @@ try: inputs, outputs, tags = simple_read_data(args.testfile) curr_out = inflect(inputs, tags, language=language) - correct = [o==c for o,c in zip(outputs,curr_out)] + correct = [o == c for o, c in zip(outputs, curr_out)] print(f"Accuracy: {float(sum(correct))/len(correct)}") except: print("dangit") - diff --git a/src/__init__.py b/unimorph_inflect/__init__.py similarity index 100% rename from src/__init__.py rename to unimorph_inflect/__init__.py diff --git a/src/inflection.py b/unimorph_inflect/inflection.py similarity index 100% rename from src/inflection.py rename to unimorph_inflect/inflection.py diff --git a/src/myutil.py b/unimorph_inflect/myutil.py similarity index 100% rename from src/myutil.py rename to unimorph_inflect/myutil.py diff --git a/utils/resources.py b/utils/resources.py index 03780c0..97b8324 100644 --- a/utils/resources.py +++ b/utils/resources.py @@ -11,40 +11,115 @@ # set home dir for default HOME_DIR = str(Path.home()) -DEFAULT_MODEL_DIR = os.path.join(HOME_DIR, 'unimorph_inflect_resources') +DEFAULT_MODEL_DIR = os.path.join(HOME_DIR, "unimorph_inflect_resources") DEFAULT_MODELS_URL = "https://github.com/antonisa/unimorph_inflect/raw/master/models" -DEFAULT_DOWNLOAD_VERSION = 'latest' +DEFAULT_DOWNLOAD_VERSION = "latest" # list of supported language names -languages = sorted(['ell', 'ell2', 'eng', 'rus', 'deu', 'cat', 'spa', 'hin', 'isl', 'hye', 'fra', 'ita', 'hbs', 'gle', 'bul', 'ady', 'ang', 'swe', 'sqi', 'ast', 'bel', 'est', 'osx', 'fas', 'ukr', 'urd', 'kat', 'nld', 'eus', 'lav', 'zul', 'que', 'dan', 'bak', 'pol', 'ara', 'heb', 'fao', 'gal', 'grc', 'ces', 'lit','nav', 'vec', 'dsb', 'por', 'ron', 'tur', 'hun', 'nob', 'san', 'cym', 'mkd', 'swc', 'slv', 'ben']) +languages = sorted( + [ + "ell", + "ell2", + "eng", + "rus", + "deu", + "cat", + "spa", + "hin", + "isl", + "hye", + "fra", + "ita", + "hbs", + "gle", + "bul", + "ady", + "ang", + "swe", + "sqi", + "ast", + "bel", + "est", + "osx", + "fas", + "ukr", + "urd", + "kat", + "nld", + "eus", + "lav", + "zul", + "que", + "dan", + "bak", + "pol", + "ara", + "heb", + "fao", + "gal", + "grc", + "ces", + "lit", + "nav", + "vec", + "dsb", + "por", + "ron", + "tur", + "hun", + "nob", + "san", + "cym", + "mkd", + "swc", + "slv", + "ben", + ] +) supported_tagset = {} -supported_tagset['eng'] = ["V;3;SG;PRS", "V;NFIN", "V;PST", "V;V.PTCP;PRS", "V;V.PTCP;PST"] +supported_tagset["eng"] = [ + "V;3;SG;PRS", + "V;NFIN", + "V;PST", + "V;V.PTCP;PRS", + "V;V.PTCP;PST", +] # download a dynet model zip file -def download_dynet_model(lang_name, resource_dir=None, should_unzip=True, confirm_if_exists=False, force=False, - version=DEFAULT_DOWNLOAD_VERSION): +def download_dynet_model( + lang_name, + resource_dir=None, + should_unzip=True, + confirm_if_exists=False, + force=False, + version=DEFAULT_DOWNLOAD_VERSION, +): # ask if user wants to download - if resource_dir is not None and os.path.exists(os.path.join(resource_dir, f"{lang_name}")): + if resource_dir is not None and os.path.exists( + os.path.join(resource_dir, f"{lang_name}") + ): if confirm_if_exists: print("") - print(f"The model directory already exists at \"{resource_dir}/{lang_name}\". Do you want to download the models again? [y/N]") - should_download = 'y' if force else input() - should_download = should_download.strip().lower() in ['yes', 'y'] + print( + f'The model directory already exists at "{resource_dir}/{lang_name}". Do you want to download the models again? [y/N]' + ) + should_download = "y" if force else input() + should_download = should_download.strip().lower() in ["yes", "y"] else: should_download = False else: - print('Would you like to download the models for: '+lang_name+' now? (Y/n)') - should_download = 'y' if force else input() - should_download = should_download.strip().lower() in ['yes', 'y', ''] + print("Would you like to download the models for: " + lang_name + " now? (Y/n)") + should_download = "y" if force else input() + should_download = should_download.strip().lower() in ["yes", "y", ""] if should_download: # set up data directory if resource_dir is None: - print('') - print('Default download directory: ' + DEFAULT_MODEL_DIR) - print('Hit enter to continue or type an alternate directory.') - where_to_download = '' if force else input() - if where_to_download != '': + print("") + print("Default download directory: " + DEFAULT_MODEL_DIR) + print("Hit enter to continue or type an alternate directory.") + where_to_download = "" if force else input() + if where_to_download != "": download_dir = where_to_download else: download_dir = DEFAULT_MODEL_DIR @@ -52,46 +127,58 @@ def download_dynet_model(lang_name, resource_dir=None, should_unzip=True, confir download_dir = resource_dir if not os.path.exists(download_dir): os.makedirs(download_dir) - model_zip_file_name = f'{lang_name}.zip' - download_url = f'{DEFAULT_MODELS_URL}/{version}/{model_zip_file_name}' + model_zip_file_name = f"{lang_name}.zip" + download_url = f"{DEFAULT_MODELS_URL}/{version}/{model_zip_file_name}" download_file_path = os.path.join(download_dir, model_zip_file_name) - print(f'Downloading models for: {lang_name} from {download_url}') - print('Download location: '+download_file_path) + print(f"Downloading models for: {lang_name} from {download_url}") + print("Download location: " + download_file_path) # initiate download r = requests.get(download_url, stream=True) - with open(download_file_path, 'wb') as f: - file_size = int(r.headers.get('content-length')) + with open(download_file_path, "wb") as f: + file_size = int(r.headers.get("content-length")) default_chunk_size = 67108864 - with tqdm(total=file_size, unit='B', unit_scale=True) as pbar: + with tqdm(total=file_size, unit="B", unit_scale=True) as pbar: for chunk in r.iter_content(chunk_size=default_chunk_size): if chunk: f.write(chunk) f.flush() pbar.update(len(chunk)) # unzip models file - print('') - print('Download complete. Models saved to: '+download_file_path) + print("") + print("Download complete. Models saved to: " + download_file_path) if should_unzip: unzip_dynet_model(lang_name, download_file_path, download_dir) # remove the zipe file print("Cleaning up...", end="") os.remove(download_file_path) - print('Done.') + print("Done.") # unzip a dynet models zip file def unzip_dynet_model(lang_name, zip_file_src, zip_file_target): - print('Extracting models file for: '+lang_name) + print("Extracting models file for: " + lang_name) with zipfile.ZipFile(zip_file_src, "r") as zip_ref: zip_ref.extractall(zip_file_target) # main download function -def download(download_language, resource_dir=None, confirm_if_exists=False, force=False, version=DEFAULT_DOWNLOAD_VERSION): +def download( + download_language, + resource_dir=None, + confirm_if_exists=False, + force=False, + version=DEFAULT_DOWNLOAD_VERSION, +): if download_language in languages: - download_dynet_model(download_language, resource_dir=resource_dir, confirm_if_exists=confirm_if_exists, force=force, - version=version) + download_dynet_model( + download_language, + resource_dir=resource_dir, + confirm_if_exists=confirm_if_exists, + force=force, + version=version, + ) else: - raise ValueError(f'The language "{download_language}" is not currently supported by this function. Please try again with other languages.') - + raise ValueError( + f'The language "{download_language}" is not currently supported by this function. Please try again with other languages.' + )