diff --git a/docs/filtering_configuration.md b/docs/filtering_configuration.md index 026dbc0bc..a61db7a60 100644 --- a/docs/filtering_configuration.md +++ b/docs/filtering_configuration.md @@ -46,3 +46,41 @@ packages = example1 example2>=1.4.2,<1.9,!=1.5.*,!=1.6.* ``` + +### Prerelease filtering + +Bandersnatch includes a plugin to filter our pre-releases of packages. To enable this plugin simply add `prerelease_release` to the enabled plugins list. + +``` ini +[blacklist] +plugins = + prerelease_release +``` + +### Regex filtering + +Advanced users who would like finer control over which packages and releases to filter can use the regex Bandersnatch plugin. + +This plugin allows arbitrary regular expressions to be defined in the configuration, any package name or release version that matches will *not* be downloaded. + +The plugin can be activated for packages and releases separately. For example to activate the project regex filter simply add it to the configuration as before: + +``` ini +[blacklist] +plugins = + regex_project +``` + +If you'd like to filter releases using the regex filter use `regex_release` instead. + +The regex plugin requires an extra section in the config to define the actual patterns to used for filtering: + +``` ini +[filter_regex] +packages = + .+-evil$ +releases = + .+alpha\d$ +``` + +Note the same `filter_regex` section may include a `packages` and a `releases` entry with any number of regular expressions. diff --git a/setup.py b/setup.py index 004105c99..a946afdc4 100755 --- a/setup.py +++ b/setup.py @@ -30,8 +30,11 @@ [bandersnatch_filter_plugins.project] blacklist_project = bandersnatch_filter_plugins.blacklist_name:BlacklistProject whitelist_project = bandersnatch_filter_plugins.whitelist_name:WhitelistProject + regex_project = bandersnatch_filter_plugins.regex_name:RegexProjectFilter [bandersnatch_filter_plugins.release] blacklist_release = bandersnatch_filter_plugins.blacklist_name:BlacklistRelease + regex_release = bandersnatch_filter_plugins.regex_name:RegexReleaseFilter + prerelease_release = bandersnatch_filter_plugins.prerelease_name:PreReleaseFilter [console_scripts] bandersnatch = bandersnatch.main:main [zc.buildout] diff --git a/src/bandersnatch/filter.py b/src/bandersnatch/filter.py index fb2943850..057106ce7 100644 --- a/src/bandersnatch/filter.py +++ b/src/bandersnatch/filter.py @@ -110,7 +110,7 @@ def filter_project_plugins() -> Iterable[Filter]: Returns ------- list of bandersnatch.filter.Filter: - List of objects drived from the bandersnatch.filter.Filter class + List of objects derived from the bandersnatch.filter.Filter class """ return load_filter_plugins("bandersnatch_filter_plugins.project") @@ -122,6 +122,6 @@ def filter_release_plugins() -> Iterable[Filter]: Returns ------- list of bandersnatch.filter.Filter: - List of objects drived from the bandersnatch.filter.Filter class + List of objects derived from the bandersnatch.filter.Filter class """ return load_filter_plugins("bandersnatch_filter_plugins.release") diff --git a/src/bandersnatch/package.py b/src/bandersnatch/package.py index 5987c1dc2..06dee00f1 100644 --- a/src/bandersnatch/package.py +++ b/src/bandersnatch/package.py @@ -147,15 +147,15 @@ def sync(self, stop_on_error=False, attempts=3): def _filter_releases(self): """ - Run the release filtering plugins and remove any packages from the - packages_to_sync that match any filters. + Run the release filtering plugins and remove any releases from + `releases` that match any filters. """ versions = list(self.releases.keys()) for version in versions: - filter = False + filter_ = False for plugin in filter_release_plugins(): - filter = filter or plugin.check_match(name=self.name, version=version) - if filter: + filter_ = filter_ or plugin.check_match(name=self.name, version=version) + if filter_: del (self.releases[version]) # TODO: async def once we go full asyncio - Have concurrency at the diff --git a/src/bandersnatch/tests/test__bandersnatch_plugins__blacklist_name.py b/src/bandersnatch/tests/plugins/test_blacklist_name.py similarity index 89% rename from src/bandersnatch/tests/test__bandersnatch_plugins__blacklist_name.py rename to src/bandersnatch/tests/plugins/test_blacklist_name.py index fdb9d2898..a38f11cae 100644 --- a/src/bandersnatch/tests/test__bandersnatch_plugins__blacklist_name.py +++ b/src/bandersnatch/tests/plugins/test_blacklist_name.py @@ -7,6 +7,7 @@ from bandersnatch.configuration import BandersnatchConfig from bandersnatch.master import Master from bandersnatch.mirror import Mirror +from bandersnatch.package import Package class TestBlacklistProject(TestCase): @@ -191,3 +192,26 @@ def test__plugin__loads__default(self): plugins = bandersnatch.filter.filter_release_plugins() names = [plugin.name for plugin in plugins] self.assertIn("blacklist_release", names) + + def test__filter__matches__release(self): + with open("test.conf", "w") as testconfig_handle: + testconfig_handle.write( + """\ +[blacklist] +plugins = + blacklist_release +packages = + foo==1.2.0 +""" + ) + instance = BandersnatchConfig() + instance.config_file = "test.conf" + instance.load_configuration() + + mirror = Mirror(".", Master(url="https://foo.bar.com")) + pkg = Package("foo", 1, mirror) + pkg.releases = {"1.2.0": {}, "1.2.1": {}} + + pkg._filter_releases() + + self.assertEqual(pkg.releases, {"1.2.1": {}}) diff --git a/src/bandersnatch/tests/plugins/test_prerelease_name.py b/src/bandersnatch/tests/plugins/test_prerelease_name.py new file mode 100644 index 000000000..65cd37671 --- /dev/null +++ b/src/bandersnatch/tests/plugins/test_prerelease_name.py @@ -0,0 +1,91 @@ +import os +import re +from collections import defaultdict +from tempfile import TemporaryDirectory +from unittest import TestCase + +import bandersnatch.filter +from bandersnatch.configuration import BandersnatchConfig +from bandersnatch.master import Master +from bandersnatch.mirror import Mirror +from bandersnatch.package import Package +from bandersnatch_filter_plugins import prerelease_name + + +def _mock_config(contents, filename="test.conf"): + """ + Creates a config file with contents and loads them into a + BandersnatchConfig instance. + """ + with open(filename, "w") as fd: + fd.write(contents) + + instance = BandersnatchConfig() + instance.config_file = filename + instance.load_configuration() + return instance + + +class BasePluginTestCase(TestCase): + + tempdir = None + cwd = None + + def setUp(self): + self.cwd = os.getcwd() + self.tempdir = TemporaryDirectory() + bandersnatch.filter.loaded_filter_plugins = defaultdict(list) + os.chdir(self.tempdir.name) + + def tearDown(self): + if self.tempdir: + os.chdir(self.cwd) + self.tempdir.cleanup() + self.tempdir = None + + +class TestRegexReleaseFilter(BasePluginTestCase): + + config_contents = """\ +[blacklist] +plugins = + prerelease_release +""" + + def test_plugin_includes_predefined_patterns(self): + _mock_config(self.config_contents) + + plugins = bandersnatch.filter.filter_release_plugins() + + assert any( + type(plugin) == prerelease_name.PreReleaseFilter for plugin in plugins + ) + plugin = next( + plugin + for plugin in plugins + if type(plugin) == prerelease_name.PreReleaseFilter + ) + expected_patterns = [ + re.compile(pattern_string) for pattern_string in plugin.PRERELEASE_PATTERNS + ] + assert plugin.patterns == expected_patterns + + def test_plugin_check_match(self): + _mock_config(self.config_contents) + + bandersnatch.filter.filter_release_plugins() + + mirror = Mirror(".", Master(url="https://foo.bar.com")) + pkg = Package("foo", 1, mirror) + pkg.releases = { + "1.2.0alpha1": {}, + "1.2.0a2": {}, + "1.2.0beta1": {}, + "1.2.0b2": {}, + "1.2.0rc1": {}, + "1.2.0": {}, + } + + pkg._filter_releases() + + assert pkg.releases == {"1.2.0": {}} diff --git a/src/bandersnatch/tests/plugins/test_regex_name.py b/src/bandersnatch/tests/plugins/test_regex_name.py new file mode 100644 index 000000000..0a2a43f85 --- /dev/null +++ b/src/bandersnatch/tests/plugins/test_regex_name.py @@ -0,0 +1,122 @@ +import os +import re +from collections import defaultdict +from tempfile import TemporaryDirectory +from unittest import TestCase + +import bandersnatch.filter +from bandersnatch.configuration import BandersnatchConfig +from bandersnatch.master import Master +from bandersnatch.mirror import Mirror +from bandersnatch.package import Package +from bandersnatch_filter_plugins import regex_name + + +def _mock_config(contents, filename="test.conf"): + """ + Creates a config file with contents and loads them into a + BandersnatchConfig instance. + """ + with open(filename, "w") as fd: + fd.write(contents) + + instance = BandersnatchConfig() + instance.config_file = filename + instance.load_configuration() + return instance + + +class BasePluginTestCase(TestCase): + + tempdir = None + cwd = None + + def setUp(self): + self.cwd = os.getcwd() + self.tempdir = TemporaryDirectory() + bandersnatch.filter.loaded_filter_plugins = defaultdict(list) + os.chdir(self.tempdir.name) + + def tearDown(self): + if self.tempdir: + os.chdir(self.cwd) + self.tempdir.cleanup() + self.tempdir = None + + +class TestRegexReleaseFilter(BasePluginTestCase): + + config_contents = """\ +[blacklist] +plugins = + regex_release + +[filter_regex] +releases = + .+rc\\d$ + .+alpha\\d$ +""" + + def test_plugin_compiles_patterns(self): + _mock_config(self.config_contents) + + plugins = bandersnatch.filter.filter_release_plugins() + + assert any(type(plugin) == regex_name.RegexReleaseFilter for plugin in plugins) + plugin = next( + plugin + for plugin in plugins + if type(plugin) == regex_name.RegexReleaseFilter + ) + assert plugin.patterns == [re.compile(r".+rc\d$"), re.compile(r".+alpha\d$")] + + def test_plugin_check_match(self): + _mock_config(self.config_contents) + + bandersnatch.filter.filter_release_plugins() + + mirror = Mirror(".", Master(url="https://foo.bar.com")) + pkg = Package("foo", 1, mirror) + pkg.releases = {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}} + + pkg._filter_releases() + + assert pkg.releases == {"foo-1.2.0": {}} + + +class TestRegexProjectFilter(BasePluginTestCase): + + config_contents = """\ +[blacklist] +plugins = + regex_project + +[filter_regex] +packages = + .+-evil$ + .+-neutral$ +""" + + def test_plugin_compiles_patterns(self): + _mock_config(self.config_contents) + + plugins = bandersnatch.filter.filter_project_plugins() + + assert any(type(plugin) == regex_name.RegexProjectFilter for plugin in plugins) + plugin = next( + plugin + for plugin in plugins + if type(plugin) == regex_name.RegexProjectFilter + ) + assert plugin.patterns == [re.compile(r".+-evil$"), re.compile(r".+-neutral$")] + + def test_plugin_check_match(self): + _mock_config(self.config_contents) + + bandersnatch.filter.filter_release_plugins() + + mirror = Mirror(".", Master(url="https://foo.bar.com")) + mirror.packages_to_sync = {"foo-good": {}, "foo-evil": {}, "foo-neutral": {}} + mirror._filter_packages() + + assert list(mirror.packages_to_sync.keys()) == ["foo-good"] diff --git a/src/bandersnatch/tests/test__bandersnatch_plugins__whitelist_name.py b/src/bandersnatch/tests/plugins/test_whitelist_name.py similarity index 100% rename from src/bandersnatch/tests/test__bandersnatch_plugins__whitelist_name.py rename to src/bandersnatch/tests/plugins/test_whitelist_name.py diff --git a/src/bandersnatch_filter_plugins/blacklist_name.py b/src/bandersnatch_filter_plugins/blacklist_name.py index 8cc5d681e..39c87c87c 100644 --- a/src/bandersnatch_filter_plugins/blacklist_name.py +++ b/src/bandersnatch_filter_plugins/blacklist_name.py @@ -125,7 +125,7 @@ def _determine_filtered_package_requirements(self): def check_match(self, **kwargs): """ - Check if the package name and version matches against a blacklisted + Check if the package name and version matches against a blacklisted package version specifier. Parameters diff --git a/src/bandersnatch_filter_plugins/prerelease_name.py b/src/bandersnatch_filter_plugins/prerelease_name.py new file mode 100644 index 000000000..3f69c9bf5 --- /dev/null +++ b/src/bandersnatch_filter_plugins/prerelease_name.py @@ -0,0 +1,43 @@ +import logging +import re + +from bandersnatch.filter import FilterReleasePlugin + +logger = logging.getLogger("bandersnatch") + + +class PreReleaseFilter(FilterReleasePlugin): + """ + Filters releases considered pre-releases. + """ + + name = "prerelease_release" + PRERELEASE_PATTERNS = (r".+rc\d$", r".+a(lpha)?\d$", r".+b(eta)?\d$") + + def initialize_plugin(self): + """ + Initialize the plugin reading patterns from the config. + """ + self.patterns = [ + re.compile(pattern_string) for pattern_string in self.PRERELEASE_PATTERNS + ] + + logger.info(f"Initialized prerelease plugin with {self.patterns}") + + def check_match(self, name, version): + """ + Check if a release version matches any of the specificed patterns. + + Parameters + ========== + name: str + Release name + version: str + Release version + + Returns + ======= + bool: + True if it matches, False otherwise. + """ + return any(pattern.match(version) for pattern in self.patterns) diff --git a/src/bandersnatch_filter_plugins/regex_name.py b/src/bandersnatch_filter_plugins/regex_name.py new file mode 100644 index 000000000..446b3a822 --- /dev/null +++ b/src/bandersnatch_filter_plugins/regex_name.py @@ -0,0 +1,89 @@ +import logging +import re + +from bandersnatch.filter import FilterProjectPlugin, FilterReleasePlugin + +logger = logging.getLogger("bandersnatch") + + +class RegexReleaseFilter(FilterReleasePlugin): + """ + Filters releases based on regex patters defined by the user. + """ + + name = "regex_release" + + def initialize_plugin(self): + """ + Initialize the plugin reading patterns from the config. + """ + # TODO: should retrieving the plugin's config be part of the base class? + try: + config = self.configuration["filter_regex"]["releases"] + except KeyError: + self.patterns = [] + else: + pattern_strings = [pattern for pattern in config.split("\n") if pattern] + self.patterns = [ + re.compile(pattern_string) for pattern_string in pattern_strings + ] + + logger.info(f"Initialized regex release plugin with {self.patterns}") + + def check_match(self, name, version): + """ + Check if a release version matches any of the specificed patterns. + + Parameters + ========== + name: str + Release name + version: str + Release version + + Returns + ======= + bool: + True if it matches, False otherwise. + """ + return any(pattern.match(version) for pattern in self.patterns) + + +class RegexProjectFilter(FilterProjectPlugin): + """ + Filters projects based on regex patters defined by the user. + """ + + name = "regex_project" + + def initialize_plugin(self): + """ + Initialize the plugin reading patterns from the config. + """ + try: + config = self.configuration["filter_regex"]["packages"] + except KeyError: + self.patterns = [] + else: + pattern_strings = [pattern for pattern in config.split("\n") if pattern] + self.patterns = [ + re.compile(pattern_string) for pattern_string in pattern_strings + ] + + logger.info(f"Initialized regex release plugin with {self.patterns}") + + def check_match(self, name): + """ + Check if a release version matches any of the specificed patterns. + + Parameters + ========== + name: str + Release name + + Returns + ======= + bool: + True if it matches, False otherwise. + """ + return any(pattern.match(name) for pattern in self.patterns)