Merge pull request #995 from emnoor-reef/ls-rm-filters

Add `--include` and `--exclude` filters to the `ls` and `rm` commands

b2/_internal/_utils/uri.py

@@ -13,11 +13,13 @@
 import pathlib
 import urllib.parse
 from pathlib import Path
+from typing import Sequence
 
 from b2sdk.v2 import (
     B2Api,
     DownloadVersion,
     FileVersion,
+    Filter,
 )
 from b2sdk.v2.exception import B2Error
 
@@ -165,10 +167,10 @@ def list_file_versions_by_uri(self, uri, *args, **kwargs):
         raise NotImplementedError(f"Unsupported URI type: {type(uri)}")
 
     @list_file_versions_by_uri.register
-    def _(self, uri: B2URI, *args, **kwargs):
+    def _(self, uri: B2URI, *args, filters: Sequence[Filter] = (), **kwargs):
         bucket = self.api.get_bucket_by_name(uri.bucket_name)
         try:
-            yield from bucket.ls(uri.path, *args, **kwargs)
+            yield from bucket.ls(uri.path, *args, filters=filters, **kwargs)
         except ValueError as error:
             # Wrap these errors into B2Error. At the time of writing there's
             # exactly one – `with_wildcard` being passed without `recursive` option.

b2/_internal/console_tool.py

@@ -79,6 +79,7 @@
     EncryptionSetting,
     FileRetentionSetting,
     FileVersion,
+    Filter,
     KeepOrDeleteMode,
     LegalHold,
     LifecycleRule,
@@ -2164,17 +2165,30 @@ class AbstractLsCommand(Command, metaclass=ABCMeta):
 
     The ``--recursive`` option will descend into folders, and will select
     only files, not folders.
+
     The ``--withWildcard`` option will allow using ``*``, ``?`` and ```[]```
     characters in ``folderName`` as a greedy wildcard, single character
     wildcard and range of characters. It requires the ``--recursive`` option.
     Remember to quote ``folderName`` to avoid shell expansion.
+
+    The --include and --exclude flags can be used to filter the files returned
+    from the server using wildcards. You can specify multiple --include and --exclude filters.
+    The order of filters matters. The *last* matching filter decides whether a file
+    is included or excluded. If the given list of filters contains only INCLUDE filters,
+    then it is assumed that all files are excluded by default.
     """
 
     @classmethod
     def _setup_parser(cls, parser):
         parser.add_argument('--versions', action='store_true')
         parser.add_argument('-r', '--recursive', action='store_true')
         parser.add_argument('--withWildcard', action='store_true')
+        parser.add_argument(
+            '--include', dest='filters', action='append', type=Filter.include, default=[]
+        )
+        parser.add_argument(
+            '--exclude', dest='filters', action='append', type=Filter.exclude, default=[]
+        )
         super()._setup_parser(parser)
 
     def _print_files(self, args):
@@ -2198,6 +2212,7 @@ def _get_ls_generator(self, args):
             latest_only=not args.versions,
             recursive=args.recursive,
             with_wildcard=args.withWildcard,
+            filters=args.filters,
         )
 
     def get_b2_uri_from_arg(self, args: argparse.Namespace) -> B2URI:
@@ -2377,7 +2392,7 @@ class SubmitThread(threading.Thread):
 
         def __init__(
             self,
-            runner: Rm,
+            runner: BaseRm,
             args: argparse.Namespace,
             messages_queue: queue.Queue,
             reporter: ProgressReport,

changelog.d/+ls-rm-filters.added.md

@@ -0,0 +1 @@
+Add `--include` and `--exclude` filters to the `ls` and `rm` commands.

test/unit/test_console_tool.py

@@ -2445,6 +2445,64 @@ def test_ls_b2id(self):
                 '''
         self._run_command(['ls', f'b2id://{file_version.id_}'], expected_stdout, '', 0)
 
+    def test_ls_filters(self):
+        self._authorize_account()
+        self._create_my_bucket()
+
+        # Create some files, including files in a folder
+        bucket = self.b2_api.get_bucket_by_name('my-bucket')
+        data = UploadSourceBytes(b'test-data')
+        bucket.upload(data, 'a/test.csv')
+        bucket.upload(data, 'a/test.tsv')
+        bucket.upload(data, 'b/b/test.csv')
+        bucket.upload(data, 'c/test.csv')
+        bucket.upload(data, 'c/test.tsv')
+        bucket.upload(data, 'test.csv')
+        bucket.upload(data, 'test.tsv')
+
+        expected_stdout = '''
+            a/
+            b/
+            c/
+            test.csv
+            '''
+        self._run_command(
+            ['ls', *self.b2_uri_args('my-bucket'), '--include', '*.csv'],
+            expected_stdout,
+        )
+        self._run_command(
+            ['ls', *self.b2_uri_args('my-bucket'), '--exclude', '*.tsv'],
+            expected_stdout,
+        )
+
+        expected_stdout = '''
+            a/test.csv
+            b/b/test.csv
+            c/test.csv
+            test.csv
+            '''
+        self._run_command(
+            ['ls', *self.b2_uri_args('my-bucket'), '--recursive', '--include', '*.csv'],
+            expected_stdout,
+        )
+        self._run_command(
+            ['ls', *self.b2_uri_args('my-bucket'), '--recursive', '--exclude', '*.tsv'],
+            expected_stdout,
+        )
+
+        expected_stdout = '''
+            b/b/test.csv
+            c/test.csv
+            test.csv
+            '''
+        self._run_command(
+            [
+                'ls', *self.b2_uri_args('my-bucket'), '--recursive', '--exclude', '*', '--include',
+                '*.csv', '--exclude', 'a/*'
+            ],
+            expected_stdout,
+        )
+
 
 class TestConsoleToolWithV1(BaseConsoleToolTest):
     """These tests use v1 interface to perform various setups before running CLI commands"""
@@ -2781,6 +2839,84 @@ def test_rm_b2id(self):
         '''
         self._run_command(['ls', '--recursive', 'b2://my-bucket'], expected_stdout)
 
+    def rm_filters_helper(self, rm_args: List[str], expected_ls_stdout: str):
+        self._authorize_account()
+        self._run_command(['create-bucket', 'my-rm-bucket', 'allPublic'], 'bucket_1\n', '', 0)
+        bucket = self.b2_api.get_bucket_by_name('my-rm-bucket')
+
+        # Create some files, including files in a folder
+        data = UploadSourceBytes(b'test-data')
+        bucket.upload(data, 'a/test.csv')
+        bucket.upload(data, 'a/test.tsv')
+        bucket.upload(data, 'b/b/test.csv')
+        bucket.upload(data, 'c/test.csv')
+        bucket.upload(data, 'c/test.tsv')
+        bucket.upload(data, 'test.csv')
+        bucket.upload(data, 'test.tsv')
+        bucket.upload(data, 'test.txt')
+
+        self._run_command(
+            ['rm', '--noProgress', *self.b2_uri_args('my-rm-bucket'), *rm_args], '', '', 0
+        )
+        self._run_command(
+            ['ls', *self.b2_uri_args('my-rm-bucket'), '--recursive'],
+            expected_ls_stdout,
+        )
+
+    def test_rm_filters_include(self):
+        expected_ls_stdout = '''
+            a/test.csv
+            a/test.tsv
+            b/b/test.csv
+            c/test.csv
+            c/test.tsv
+            test.tsv
+            test.txt
+            '''
+        self.rm_filters_helper(['--include', '*.csv'], expected_ls_stdout)
+
+    def test_rm_filters_exclude(self):
+        expected_ls_stdout = '''
+            a/test.csv
+            a/test.tsv
+            b/b/test.csv
+            c/test.csv
+            c/test.tsv
+            test.csv
+            '''
+        self.rm_filters_helper(['--exclude', '*.csv'], expected_ls_stdout)
+
+    def test_rm_filters_include_recursive(self):
+        expected_ls_stdout = '''
+            a/test.tsv
+            c/test.tsv
+            test.tsv
+            test.txt
+            '''
+        self.rm_filters_helper(['--recursive', '--include', '*.csv'], expected_ls_stdout)
+
+    def test_rm_filters_exclude_recursive(self):
+        expected_ls_stdout = '''
+            a/test.csv
+            b/b/test.csv
+            c/test.csv
+            test.csv
+            '''
+        self.rm_filters_helper(['--recursive', '--exclude', '*.csv'], expected_ls_stdout)
+
+    def test_rm_filters_mixed(self):
+        expected_ls_stdout = '''
+            a/test.csv
+            a/test.tsv
+            c/test.tsv
+            test.tsv
+            test.txt
+            '''
+        self.rm_filters_helper(
+            ['--recursive', '--exclude', '*', '--include', '*.csv', '--exclude', 'a/*'],
+            expected_ls_stdout
+        )
+
 
 class TestVersionConsoleTool(BaseConsoleToolTest):
     def test_version(self):