From 9dcb5e402ab7ba3844022e28a2485bf70ea229ef Mon Sep 17 00:00:00 2001 From: sgoodm Date: Fri, 10 Feb 2023 16:14:00 -0500 Subject: [PATCH] Add type hints to pm25 --- cru_ts/create_env.sh | 3 --- cru_ts/environment.yml | 36 ------------------------------------ cru_ts/export_env.sh | 3 --- cru_ts/flow.py | 6 ++++-- cru_ts/jobscript | 27 --------------------------- cru_ts/main.py | 2 +- pm25/flow.py | 17 +++++++++++++++-- pm25/main.py | 6 +++--- 8 files changed, 23 insertions(+), 77 deletions(-) delete mode 100755 cru_ts/create_env.sh delete mode 100644 cru_ts/environment.yml delete mode 100755 cru_ts/export_env.sh delete mode 100644 cru_ts/jobscript diff --git a/cru_ts/create_env.sh b/cru_ts/create_env.sh deleted file mode 100755 index 5f4bf757..00000000 --- a/cru_ts/create_env.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -conda env create -f environment.yml diff --git a/cru_ts/environment.yml b/cru_ts/environment.yml deleted file mode 100644 index a8660c87..00000000 --- a/cru_ts/environment.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: cru_ts_405 -channels: - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=4.5=1_gnu - - ca-certificates=2021.5.25=h06a4308_1 - - certifi=2021.5.30=py38h06a4308_0 - - ld_impl_linux-64=2.35.1=h7274673_9 - - libffi=3.3=he6710b0_2 - - libgcc-ng=9.3.0=h5101ec6_17 - - libgomp=9.3.0=h5101ec6_17 - - libstdcxx-ng=9.3.0=hd4cf53a_17 - - ncurses=6.2=he6710b0_1 - - openssl=1.1.1k=h27cfd23_0 - - pip=21.1.2=py38h06a4308_0 - - python=3.8.10=h12debd9_8 - - readline=8.1=h27cfd23_0 - - setuptools=52.0.0=py38h06a4308_0 - - sqlite=3.35.4=hdfb4753_0 - - tk=8.6.10=hbc83047_0 - - wheel=0.36.2=pyhd3eb1b0_0 - - xz=5.2.5=h7b6447c_0 - - zlib=1.2.11=h7b6447c_3 - - pip: - - affine==2.3.0 - - attrs==21.2.0 - - cftime==1.5.0 - - click==8.0.1 - - click-plugins==1.1.1 - - cligj==0.7.2 - - netcdf4==1.5.6 - - numpy==1.20.3 - - pyparsing==2.4.7 - - rasterio==1.2.4 - - snuggs==1.4.7 diff --git a/cru_ts/export_env.sh b/cru_ts/export_env.sh deleted file mode 100755 index 18307a15..00000000 --- a/cru_ts/export_env.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -conda env export > environment.yml diff --git a/cru_ts/flow.py b/cru_ts/flow.py index 6a28c1a6..d7585450 100644 --- a/cru_ts/flow.py +++ b/cru_ts/flow.py @@ -89,7 +89,9 @@ def cru_ts( class_instance = CRU_TS(cru_vers, years, raw_dir, output_dir, overwrite_download, overwrite_unzip, overwrite_processing) + if task_runner != 'hpc': os.chdir(tmp_dir) - - class_instance.run(backend=backend, task_runner=task_runner, run_parallel=run_parallel, max_workers=max_workers, log_dir=timestamp_log_dir, cluster_kwargs=cluster_kwargs) + class_instance.run(backend=backend, task_runner=task_runner, run_parallel=run_parallel, max_workers=max_workers, log_dir=timestamp_log_dir) + else: + class_instance.run(backend=backend, task_runner=task_runner, run_parallel=run_parallel, max_workers=max_workers, log_dir=timestamp_log_dir, cluster=cluster, cluster_kwargs=hpc_cluster_kwargs) diff --git a/cru_ts/jobscript b/cru_ts/jobscript deleted file mode 100644 index 19c7065c..00000000 --- a/cru_ts/jobscript +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/tcsh -#PBS -N asg:cru_ts -#PBS -l nodes=1:c18c:ppn=16 -#PBS -l walltime=24:00:00 -#PBS -j oe - -# init conda within new shell for job - -# vortex [nodespec c18a or c18c for vortex-alpha, ppn up to 12 or 16 for vortex / alpha] -source "/usr/local/anaconda3-2021.05/etc/profile.d/conda.csh" -module load anaconda3/2021.05 - -# hima [nodespec: hima:nogpu, ppn up to 32] -# source "/usr/local/anaconda3-4.4.0/etc/profile.d/conda.csh" -# module load anaconda3/4.4.0 - - -unsetenv PYTHONPATH - - -conda activate cru_ts_405 - -setenv src_dir /sciclone/aiddata10/geo/master/source/geo-datasets -# setenv src_dir /sciclone/home20/smgoodman/geo-datasets - - -python ${src_dir}/cru_ts/extract_data.py \ No newline at end of file diff --git a/cru_ts/main.py b/cru_ts/main.py index 503f49eb..14ef592a 100644 --- a/cru_ts/main.py +++ b/cru_ts/main.py @@ -35,7 +35,6 @@ from configparser import ConfigParser from typing import List - import rasterio import numpy as np @@ -43,6 +42,7 @@ from dataset import Dataset + class CRU_TS(Dataset): name = "Climatic Research Unit gridded Time Series" diff --git a/pm25/flow.py b/pm25/flow.py index 059589d0..ee3d9b09 100644 --- a/pm25/flow.py +++ b/pm25/flow.py @@ -1,8 +1,8 @@ import os -import sys from pathlib import Path from datetime import datetime from configparser import ConfigParser +from typing import List, Literal from prefect import flow from prefect.filesystems import GitHub @@ -21,7 +21,20 @@ @flow -def pm25(raw_dir, output_dir, box_config_path, version, years, overwrite_downloads, verify_existing_downloads, overwrite_processing, backend, task_runner, run_parallel, max_workers, log_dir): +def pm25( + raw_dir: str, + output_dir: str, + box_config_path: str, + version: str, + years: List[int], + overwrite_downloads: bool, + verify_existing_downloads: bool, + overwrite_processing: bool, + backend: Literal["local", "mpi", "prefect"], + task_runner: Literal["sequential", "concurrent", "dask", "hpc"], + run_parallel: bool, + max_workers: int, + log_dir: str): timestamp = datetime.today() time_str = timestamp.strftime("%Y_%m_%d_%H_%M") diff --git a/pm25/main.py b/pm25/main.py index 1824e915..0587c352 100644 --- a/pm25/main.py +++ b/pm25/main.py @@ -88,9 +88,9 @@ def __init__(self, box_config_path: str, version: str, years: list, - overwrite_downloads=True, - verify_existing_downloads=True, - overwrite_processing=False,): + overwrite_downloads: bool, + verify_existing_downloads: bool, + overwrite_processing: bool,): self.version = version