diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dccdb5e..e9b98fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,9 +1,9 @@ name: test on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: build: @@ -11,11 +11,11 @@ jobs: strategy: fail-fast: true matrix: - python-version: [ 3.9] + python-version: [ '3.9', '3.10', '3.11', '3.12' ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5da3996..7ba4539 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,11 +5,12 @@ repos: # - id: double-quote-string-fixer # for single quotes: uncomment and add black config “skip-string-normalization” - id: trailing-whitespace - id: end-of-file-fixer - - repo: https://github.com/PyCQA/docformatter - rev: v1.7.5 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.6 hooks: - - id: docformatter - args: ["--wrap-summaries=100", "--wrap-descriptions=100"] + - id: ruff + args: [ --fix ] + - id: ruff-format - repo: https://github.com/psf/black rev: 24.4.2 hooks: diff --git a/strato/backends/_gcp.py b/strato/backends/_gcp.py index 35f92a8..ab46db6 100644 --- a/strato/backends/_gcp.py +++ b/strato/backends/_gcp.py @@ -1,20 +1,21 @@ import shutil -from subprocess import check_call +from subprocess import DEVNULL, check_call class GCPBackend: def __init__(self): - if shutil.which("gsutil") is None: + if shutil.which("gcloud") is None: raise Exception("google-cloud-sdk is not installed!") self._backend = "gcp" self._call_prefix = [ - "gsutil", - "-q", - "-o", - "GSUtil:parallel_composite_upload_threshold=150M", + "gcloud", + "storage", + "--no-user-output-enabled", + # "-o", + # "GSUtil:parallel_composite_upload_threshold=150M", ] - def copy(self, recursive, parallel, ionice, filenames, quiet, dryrun): + def copy(self, recursive, ionice, filenames, quiet, dryrun): # FIXME make local target directories to mimic behavior of aws and local backends call_args = ( ["ionice", "-c", "2", "-n", "7"] @@ -22,8 +23,6 @@ def copy(self, recursive, parallel, ionice, filenames, quiet, dryrun): else [] ) call_args += self._call_prefix - if parallel: - call_args.append("-m") call_args.append("cp") if recursive: call_args.append("-r") @@ -33,7 +32,7 @@ def copy(self, recursive, parallel, ionice, filenames, quiet, dryrun): if not dryrun: check_call(call_args) - def sync(self, parallel, ionice, source, target, quiet, dryrun): + def sync(self, ionice, source, target, quiet, dryrun): # If target folder is local. if len(target.split("://")) == 1: import os @@ -47,18 +46,14 @@ def sync(self, parallel, ionice, source, target, quiet, dryrun): else [] ) call_args += self._call_prefix - if parallel: - call_args.append("-m") - call_args.extend(["rsync", "-d", "-r", source, target]) + call_args.extend(["rsync", "--delete-unmatched-destination-objects", "-r", source, target]) if not quiet or dryrun: print(" ".join(call_args)) if not dryrun: check_call(call_args) - def delete(self, recursive, parallel, filenames, quiet, dryrun): + def delete(self, recursive, filenames, quiet, dryrun): call_args = self._call_prefix.copy() - if parallel: - call_args.append("-m") call_args.append("rm") if recursive: call_args.append("-r") @@ -70,11 +65,5 @@ def delete(self, recursive, parallel, filenames, quiet, dryrun): def stat(self, filename): assert filename.startswith("gs://"), "Must be a GS URI!" - is_folder = True if filename[-1] == "/" else False - - if is_folder: - call_args = ["gsutil", "-q", "stat", filename + "*"] - else: - call_args = ["gsutil", "-q", "stat", filename] - - check_call(call_args) + call_args = ["gcloud", "storage", "ls", filename] + check_call(call_args, stdout=DEVNULL) diff --git a/strato/backends/_local.py b/strato/backends/_local.py index 57ad4c8..98de2a4 100644 --- a/strato/backends/_local.py +++ b/strato/backends/_local.py @@ -57,7 +57,8 @@ def sync(self, ionice, source, target, quiet, dryrun): if shutil.which("rsync") is None: raise Exception("rsync is not installed!") # target = os.path.dirname(target) - os.makedirs(target, exist_ok=True) + if not dryrun: + os.makedirs(target, exist_ok=True) call_args = ( ["ionice", "-c", "2", "-n", "7"] if ionice and (shutil.which("ionice")) is not None diff --git a/strato/commands/cp.py b/strato/commands/cp.py index aba0032..72d0f08 100644 --- a/strato/commands/cp.py +++ b/strato/commands/cp.py @@ -10,17 +10,17 @@ strato cp s3://my-bucket/source_folder/*.zip /target_folder/ # GCP upload - strato cp -m -r --ionice file1 folder2 gs://my-bucket/target_folder/ + strato cp -r --ionice file1 folder2 gs://my-bucket/target_folder/ # GCP download mkdir /target_folder - strato cp -m gs://my-bucket/source_folder/*.zip /target_folder/ + strato cp gs://my-bucket/source_folder/*.zip /target_folder/ # On local machine strato cp -r file1 folder2 /target_folder/ """ -def copy_files(recursive, parallel, ionice, filenames, profile, quiet, dryrun): +def copy_files(recursive, ionice, filenames, profile, quiet, dryrun): backend = get_backend(filenames) if backend == "aws": @@ -32,7 +32,7 @@ def copy_files(recursive, parallel, ionice, filenames, profile, quiet, dryrun): from strato.backends import GCPBackend be = GCPBackend() - be.copy(recursive, parallel, ionice, filenames, quiet, dryrun) + be.copy(recursive, ionice, filenames, quiet, dryrun) else: from strato.backends import LocalBackend @@ -57,7 +57,7 @@ def main(argsv): "-m", dest="parallel", action="store_true", - help="Run operations in parallel. Only available for GCP backend.", + help=argparse.SUPPRESS, ) parser.add_argument( "--ionice", @@ -73,7 +73,10 @@ def main(argsv): help="AWS profile. Only works for aws backend, and use the default profile if not provided.", ) parser.add_argument( - "--quiet", dest="quiet", action="store_true", help="Hide the underlying command." + "--quiet", + dest="quiet", + action="store_true", + help="Hide the underlying command.", ) parser.add_argument( "--dryrun", @@ -88,7 +91,6 @@ def main(argsv): args = parser.parse_args(argsv) copy_files( args.recursive, - args.parallel, args.ionice, args.files, args.profile, diff --git a/strato/commands/rm.py b/strato/commands/rm.py index 58c9ac3..e8a7f1b 100644 --- a/strato/commands/rm.py +++ b/strato/commands/rm.py @@ -5,12 +5,12 @@ example_text = """Examples: strato rm s3://my-bucket/file1 s3://my-bucket/folder2/ - strato rm -m gs://my-bucket/file1 gs://my-bucket/folder2 gs://my-bucket/folder3/*.zip + strato rm gs://my-bucket/file1 gs://my-bucket/folder2 gs://my-bucket/folder3/*.zip strato rm file1 folder2 """ -def delete_files(recursive, parallel, filenames, profile, quiet, dryrun): +def delete_files(recursive, filenames, profile, quiet, dryrun): backend = get_backend(filenames) if backend == "aws": @@ -22,7 +22,7 @@ def delete_files(recursive, parallel, filenames, profile, quiet, dryrun): from strato.backends import GCPBackend be = GCPBackend() - be.delete(recursive, parallel, filenames, quiet, dryrun) + be.delete(recursive, filenames, quiet, dryrun) else: from strato.backends import LocalBackend @@ -54,10 +54,13 @@ def main(argsv): "-m", dest="parallel", action="store_true", - help="Run operations in parallel. Only available for GCP backend.", + help=argparse.SUPPRESS, ) parser.add_argument( - "--quiet", dest="quiet", action="store_true", help="Hide the underlying command." + "--quiet", + dest="quiet", + action="store_true", + help="Hide the underlying command.", ) parser.add_argument( "--dryrun", @@ -69,4 +72,4 @@ def main(argsv): ) args = parser.parse_args(argsv) - delete_files(args.recursive, args.parallel, args.files, args.profile, args.quiet, args.dryrun) + delete_files(args.recursive, args.files, args.profile, args.quiet, args.dryrun) diff --git a/strato/commands/sync.py b/strato/commands/sync.py index 08e8a42..0d380a2 100644 --- a/strato/commands/sync.py +++ b/strato/commands/sync.py @@ -5,12 +5,12 @@ example_text = """Examples: strato sync source_folder s3://my-bucket/target_folder - strato sync -m --ionice source_folder gs://my-bucket/target_folder + strato sync --ionice source_folder gs://my-bucket/target_folder strato sync source_folder target_folder """ -def synchronize_folders(parallel, ionice, source, target, profile, quiet, dryrun): +def synchronize_folders(ionice, source, target, profile, quiet, dryrun): backend = get_backend([source, target]) if backend == "aws": @@ -22,7 +22,7 @@ def synchronize_folders(parallel, ionice, source, target, profile, quiet, dryrun from strato.backends import GCPBackend be = GCPBackend() - be.sync(parallel, ionice, source, target, quiet, dryrun) + be.sync(ionice, source, target, quiet, dryrun) else: from strato.backends import LocalBackend @@ -40,7 +40,7 @@ def main(argsv): "-m", dest="parallel", action="store_true", - help="Run operations in parallel. Only available for GCP backend.", + help=argparse.SUPPRESS, ) parser.add_argument( "--ionice", @@ -56,7 +56,10 @@ def main(argsv): help="AWS profile. Only works for aws backend, and use the default profile if not provided.", ) parser.add_argument( - "--quiet", dest="quiet", action="store_true", help="Hide the underlying command." + "--quiet", + dest="quiet", + action="store_true", + help="Hide the underlying command.", ) parser.add_argument( "--dryrun", @@ -68,5 +71,5 @@ def main(argsv): args = parser.parse_args(argsv) synchronize_folders( - args.parallel, args.ionice, args.source, args.target, args.profile, args.quiet, args.dryrun + args.ionice, args.source, args.target, args.profile, args.quiet, args.dryrun ) diff --git a/strato/tests/helpers.py b/strato/tests/helpers.py index 027975e..e4ee912 100644 --- a/strato/tests/helpers.py +++ b/strato/tests/helpers.py @@ -1 +1 @@ -gsutil = "gsutil -q -o GSUtil:parallel_composite_upload_threshold=150M" +gcloud = "gcloud storage --no-user-output-enabled" diff --git a/strato/tests/test_cp.py b/strato/tests/test_cp.py index 4abf9a6..b0a87e0 100644 --- a/strato/tests/test_cp.py +++ b/strato/tests/test_cp.py @@ -1,7 +1,7 @@ import pytest from strato.commands import cp -from strato.tests.helpers import gsutil +from strato.tests.helpers import gcloud def test_cp_file_aws(capsys): @@ -25,17 +25,17 @@ def test_cp_dir_aws(capsys, trailing_slash): def test_cp_file_gcp(capsys): cp.main(["file1", "gs://foo/bar/", "--dryrun"]) - assert gsutil + " cp file1 gs://foo/bar/\n" == capsys.readouterr().out + assert gcloud + " cp file1 gs://foo/bar/\n" == capsys.readouterr().out def test_cp_dir_gcp(capsys): cp.main(["dir1", "gs://foo/bar", "-r", "--dryrun"]) - assert gsutil + " cp -r dir1 gs://foo/bar\n" == capsys.readouterr().out + assert gcloud + " cp -r dir1 gs://foo/bar\n" == capsys.readouterr().out def test_cp_file_local(capsys): cp.main(["file1", "/bar/foo", "--dryrun"]) - assert "cp file1 /bar/foo\n" == capsys.readouterr().out + assert "mkdir -p /bar\ncp file1 /bar/foo\n" == capsys.readouterr().out def test_cp_dir_local(capsys): diff --git a/strato/tests/test_rm.py b/strato/tests/test_rm.py index 5fdbc23..837af74 100644 --- a/strato/tests/test_rm.py +++ b/strato/tests/test_rm.py @@ -1,5 +1,5 @@ from strato.commands import rm -from strato.tests.helpers import gsutil +from strato.tests.helpers import gcloud def test_rm_aws(capsys): @@ -16,12 +16,12 @@ def test_rm_aws_recursive(capsys): def test_rm_gcp(capsys): rm.main(["gs://foo/bar/", "--dryrun"]) - assert gsutil + " rm gs://foo/bar/\n" == capsys.readouterr().out + assert gcloud + " rm gs://foo/bar/\n" == capsys.readouterr().out def test_rm_gcp_recursive(capsys): rm.main(["gs://foo/bar/", "--dryrun", "--recursive"]) - assert gsutil + " rm -r gs://foo/bar/\n" == capsys.readouterr().out + assert gcloud + " rm -r gs://foo/bar/\n" == capsys.readouterr().out def test_rm_local(capsys): diff --git a/strato/tests/test_sync.py b/strato/tests/test_sync.py index 57f974b..8747dee 100644 --- a/strato/tests/test_sync.py +++ b/strato/tests/test_sync.py @@ -1,19 +1,22 @@ from strato.commands import sync -from strato.tests.helpers import gsutil +from strato.tests.helpers import gcloud def test_sync_aws(capsys): - sync.main(["file1", "s3://foo/bar/", "--dryrun"]) + sync.main(["folder1", "s3://foo/bar/", "--dryrun"]) assert ( - "aws s3 sync --delete --only-show-errors file1 s3://foo/bar/\n" == capsys.readouterr().out + "aws s3 sync --delete --only-show-errors folder1 s3://foo/bar/\n" == capsys.readouterr().out ) def test_sync_gcp(capsys): - sync.main(["file1", "gs://foo/bar/", "--dryrun"]) - assert gsutil + " rsync -d -r file1 gs://foo/bar/\n" == capsys.readouterr().out + sync.main(["folder1", "gs://foo/bar/", "--dryrun"]) + assert ( + gcloud + " rsync --delete-unmatched-destination-objects -r folder1 gs://foo/bar/\n" + == capsys.readouterr().out + ) def test_sync_local(capsys): - sync.main(["file1", "/bar/foo", "--dryrun"]) - assert "rsync -r --delete file1 /bar\n" == capsys.readouterr().out + sync.main(["folder1", "/bar/foo", "--dryrun"]) + assert "rsync -r --delete folder1/ /bar/foo/\n" == capsys.readouterr().out