Skip to content

Commit

Permalink
Merge pull request #1038 from reef-technologies/file-unhide
Browse files Browse the repository at this point in the history
Prepare new minor release: File unhide + TruncateOutput fix
  • Loading branch information
mjurbanski-reef authored Jul 30, 2024
2 parents 376b060 + 28ceb19 commit 9f6127f
Show file tree
Hide file tree
Showing 11 changed files with 200 additions and 15 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ env:

jobs:
lint:
timeout-minutes: 60
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -72,6 +73,7 @@ jobs:
if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} # TODO: skip this whole job instead
run: nox -vs cleanup_buckets
test:
timeout-minutes: 90
needs: cleanup_buckets
env:
B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
Expand Down Expand Up @@ -131,6 +133,7 @@ jobs:
if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' && contains(fromJSON('["3.7", "pypy3.10", "3.12"]'), matrix.python-version) }}
run: nox -vs integration -p ${{ matrix.python-version }} -- -m "require_secrets" --cleanup
test-docker:
timeout-minutes: 90
needs: cleanup_buckets
env:
B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
Expand Down Expand Up @@ -163,6 +166,7 @@ jobs:
if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
run: nox -vs docker_test -- backblazeit/b2:test
test-linux-bundle:
timeout-minutes: 90
needs: cleanup_buckets
env:
B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
Expand Down Expand Up @@ -201,6 +205,7 @@ jobs:
if-no-files-found: warn
retention-days: 7
test-windows-bundle:
timeout-minutes: 90
needs: cleanup_buckets
env:
B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
Expand Down Expand Up @@ -240,6 +245,7 @@ jobs:
if-no-files-found: warn
retention-days: 7
doc:
timeout-minutes: 30
needs: build
runs-on: ubuntu-latest
steps:
Expand Down
24 changes: 24 additions & 0 deletions b2/_internal/_cli/b2args.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ def parse_bucket_name(value: str, allow_all_buckets: bool = False) -> str:
return str(value)


def b2id_or_file_like_b2_uri_or_bucket_name(value: str) -> Union[B2URIBase, str]:
try:
bucket_name = parse_bucket_name(value)
return bucket_name
except ValueError:
return b2id_or_file_like_b2_uri(value)


B2ID_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_uri)
B2_BUCKET_URI_ARG_TYPE = wrap_with_argument_type_error(b2_bucket_uri)
B2ID_OR_B2_URI_ARG_TYPE = wrap_with_argument_type_error(parse_b2_uri)
Expand All @@ -86,6 +94,9 @@ def parse_bucket_name(value: str, allow_all_buckets: bool = False) -> str:
functools.partial(parse_b2_uri, allow_all_buckets=True)
)
B2ID_OR_FILE_LIKE_B2_URI_ARG_TYPE = wrap_with_argument_type_error(b2id_or_file_like_b2_uri)
B2ID_OR_FILE_LIKE_B2_URI_OR_BUCKET_NAME_ARG_TYPE = wrap_with_argument_type_error(
b2id_or_file_like_b2_uri_or_bucket_name
)


def add_bucket_name_argument(
Expand Down Expand Up @@ -194,5 +205,18 @@ def add_b2id_or_file_like_b2_uri_argument(parser: argparse.ArgumentParser, name=
).completer = b2uri_file_completer


def add_b2id_or_file_like_b2_uri_or_bucket_name_argument(
parser: argparse.ArgumentParser, name="B2_URI"
):
"""
Add a B2 URI pointing to a file as an argument to the parser.
"""
parser.add_argument(
name,
type=B2ID_OR_FILE_LIKE_B2_URI_OR_BUCKET_NAME_ARG_TYPE,
help="B2 URI pointing to a file, e.g. b2://yourBucket/file.txt or b2id://fileId",
).completer = b2uri_file_completer


def get_keyid_and_key_from_env_vars() -> Tuple[Optional[str], Optional[str]]:
return environ.get(B2_APPLICATION_KEY_ID_ENV_VAR), environ.get(B2_APPLICATION_KEY_ENV_VAR)
69 changes: 67 additions & 2 deletions b2/_internal/console_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@
add_b2id_or_b2_bucket_uri_argument,
add_b2id_or_b2_uri_argument,
add_b2id_or_file_like_b2_uri_argument,
add_b2id_or_file_like_b2_uri_or_bucket_name_argument,
add_b2id_uri_argument,
add_bucket_name_argument,
get_keyid_and_key_from_env_vars,
Expand Down Expand Up @@ -712,6 +713,21 @@ def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase:
return args.B2_URI


class B2URIFileOrBucketNameFileNameArgMixin:
@classmethod
def _setup_parser(cls, parser):
add_b2id_or_file_like_b2_uri_or_bucket_name_argument(parser)
parser.add_argument('fileName', nargs='?', help=argparse.SUPPRESS)
super()._setup_parser(parser)

def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URIBase | str:
if isinstance(args.B2_URI, B2URI):
return args.B2_URI

bucket_name = args.B2_URI
return bucket_name


class B2URIFileIDArgMixin:
@classmethod
def _setup_parser(cls, parser):
Expand Down Expand Up @@ -2161,6 +2177,22 @@ class FileHideBase(Command):
- **writeFiles**
"""

def _run(self, args):
b2_uri = self.get_b2_uri_from_arg(args)
if isinstance(b2_uri, B2URI):
bucket_name = b2_uri.bucket_name
file_name = b2_uri.path
else:
bucket_name = b2_uri
file_name = args.fileName

bucket = self.api.get_bucket_by_name(bucket_name)
file_info = bucket.hide_file(file_name)
self._print_json(file_info)
return 0


class HideFileBase(Command):
@classmethod
def _setup_parser(cls, parser):
add_bucket_name_argument(parser)
Expand All @@ -2174,6 +2206,33 @@ def _run(self, args):
return 0


class FileUnhideBase(Command):
"""
Delete the "hide marker" for a given file.
Requires capability:
- **listFiles**
- **deleteFiles**
and optionally:
- **bypassGovernance**
"""

@classmethod
def _setup_parser(cls, parser):
add_normalized_argument(parser, '--bypass-governance', action='store_true', default=False)
super()._setup_parser(parser)

def _run(self, args):
b2_uri = self.get_b2_uri_from_arg(args)
bucket = self.api.get_bucket_by_name(b2_uri.bucket_name)
file_id_and_name = bucket.unhide_file(b2_uri.path, args.bypass_governance)
self._print_json(file_id_and_name)
return 0


class BucketListBase(Command):
"""
List all of the buckets in the current account.
Expand Down Expand Up @@ -5073,11 +5132,17 @@ class FileCopyById(FileCopyByIdBase):


@File.subcommands_registry.register
class FileHide(FileHideBase):
class FileHide(B2URIFileOrBucketNameFileNameArgMixin, FileHideBase):
__doc__ = FileHideBase.__doc__
COMMAND_NAME = 'hide'


@File.subcommands_registry.register
class FileUnhide(B2URIFileArgMixin, FileUnhideBase):
__doc__ = FileUnhideBase.__doc__
COMMAND_NAME = 'unhide'


@File.subcommands_registry.register
class FileUpdate(FileUpdateBase):
__doc__ = FileUpdateBase.__doc__
Expand Down Expand Up @@ -5145,7 +5210,7 @@ class CopyFileById(CmdReplacedByMixin, FileCopyByIdBase):
replaced_by_cmd = (File, FileCopyById)


class HideFile(CmdReplacedByMixin, FileHideBase):
class HideFile(CmdReplacedByMixin, HideFileBase):
__doc__ = FileHideBase.__doc__
replaced_by_cmd = (File, FileHide)

Expand Down
1 change: 1 addition & 0 deletions changelog.d/+b2_file_hide.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Support new `b2_uri` and deprecated `bucket_name file_name` arguments in `b2 file hide`.
1 change: 1 addition & 0 deletions changelog.d/+b2_file_unhide.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add `b2 file unhide` command.
1 change: 1 addition & 0 deletions changelog.d/554.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Update `b2sdk` to 2.5.0, to fix `TruncatedOutput` download errors when network is congested (e.g., due use of high downloader thread count).
10 changes: 5 additions & 5 deletions pdm.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ classifiers = [
dependencies = [
"argcomplete>=2,<4",
"arrow>=1.0.2,<2.0.0",
"b2sdk>=2.4.1,<3",
"b2sdk>=2.5.0,<3",
"docutils>=0.18.1",
"idna~=3.4; platform_system == 'Java'",
"importlib-metadata>=3.3; python_version < '3.8'",
Expand Down
14 changes: 14 additions & 0 deletions test/integration/test_b2_command_line.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,20 @@ def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args, apiver_
)
should_equal(['a', 'b/1', 'b/2', 'd'], [f['fileName'] for f in list_of_files])

b2_tool.should_succeed(['file', 'unhide', f'b2://{bucket_name}/c'])

list_of_files = b2_tool.should_succeed_json(
['ls', '--json', '--recursive', *b2_uri_args(bucket_name)]
)
should_equal(['a', 'b/1', 'b/2', 'c', 'd'], [f['fileName'] for f in list_of_files])

b2_tool.should_succeed(['file', 'hide', bucket_name, 'c'])

list_of_files = b2_tool.should_succeed_json(
['ls', '--json', '--recursive', *b2_uri_args(bucket_name)]
)
should_equal(['a', 'b/1', 'b/2', 'd'], [f['fileName'] for f in list_of_files])

list_of_files = b2_tool.should_succeed_json(
['ls', '--json', '--recursive', '--versions', *b2_uri_args(bucket_name)]
)
Expand Down
4 changes: 2 additions & 2 deletions test/unit/_cli/test_autocomplete_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def test_complete_with_escaped_control_characters(
store=autocomplete_cache.HomeCachePickleStore(tmp_path),
)

with autocomplete_runner(f'b2 file hide {bucket} '):
with autocomplete_runner(f'b2 hide-file {bucket} '):
exit, argcomplete_output = argcomplete_result()
assert exit == 0
assert escaped_cc_file_name in argcomplete_output
Expand All @@ -200,7 +200,7 @@ def test_complete_with_file_suggestions(
tracker=autocomplete_cache.VersionTracker(),
store=autocomplete_cache.HomeCachePickleStore(tmp_path),
)
with autocomplete_runner(f'b2 file hide {bucket} '):
with autocomplete_runner(f'b2 hide-file {bucket} '):
exit, argcomplete_output = argcomplete_result()
assert exit == 0
assert file_name in argcomplete_output
Expand Down
Loading

0 comments on commit 9f6127f

Please sign in to comment.