Pre-release filter (#83)

* Pre-release filter

Grouped and renamed filter tests

* Fix uninitialized `patterns`

* Fix linting

* Added `RegexProjectFilter`

* Use `[filter_*]` config convention

* Add `PreReleaseFilter`

* Additional test and documentation

* Use "bandersnatch" logger

* Add test for blacklist release

Use proper release naming

docs/filtering_configuration.md

@@ -46,3 +46,41 @@ packages =
     example1
     example2>=1.4.2,<1.9,!=1.5.*,!=1.6.*
 ```
+
+### Prerelease filtering
+
+Bandersnatch includes a plugin to filter our pre-releases of packages. To enable this plugin simply add `prerelease_release` to the enabled plugins list.
+
+``` ini
+[blacklist]
+plugins =
+    prerelease_release
+```
+
+### Regex filtering
+
+Advanced users who would like finer control over which packages and releases to filter can use the regex Bandersnatch plugin.
+
+This plugin allows arbitrary regular expressions to be defined in the configuration, any package name or release version that matches will *not* be downloaded.
+
+The plugin can be activated for packages and releases separately. For example to activate the project regex filter simply add it to the configuration as before:
+
+``` ini
+[blacklist]
+plugins =
+    regex_project
+```
+
+If you'd like to filter releases using the regex filter use `regex_release` instead.
+
+The regex plugin requires an extra section in the config to define the actual patterns to used for filtering:
+
+``` ini
+[filter_regex]
+packages =
+    .+-evil$
+releases =
+    .+alpha\d$
+```
+
+Note the same `filter_regex` section may include a `packages` and a `releases` entry with any number of regular expressions.

setup.py

@@ -30,8 +30,11 @@
             [bandersnatch_filter_plugins.project]
                 blacklist_project = bandersnatch_filter_plugins.blacklist_name:BlacklistProject
                 whitelist_project = bandersnatch_filter_plugins.whitelist_name:WhitelistProject
+                regex_project = bandersnatch_filter_plugins.regex_name:RegexProjectFilter
             [bandersnatch_filter_plugins.release]
                 blacklist_release = bandersnatch_filter_plugins.blacklist_name:BlacklistRelease
+                regex_release = bandersnatch_filter_plugins.regex_name:RegexReleaseFilter
+                prerelease_release = bandersnatch_filter_plugins.prerelease_name:PreReleaseFilter
             [console_scripts]
                 bandersnatch = bandersnatch.main:main
             [zc.buildout]

src/bandersnatch/filter.py

@@ -110,7 +110,7 @@ def filter_project_plugins() -> Iterable[Filter]:
     Returns
     -------
     list of bandersnatch.filter.Filter:
-        List of objects drived from the bandersnatch.filter.Filter class
+        List of objects derived from the bandersnatch.filter.Filter class
     """
     return load_filter_plugins("bandersnatch_filter_plugins.project")
 
@@ -122,6 +122,6 @@ def filter_release_plugins() -> Iterable[Filter]:
     Returns
     -------
     list of bandersnatch.filter.Filter:
-        List of objects drived from the bandersnatch.filter.Filter class
+        List of objects derived from the bandersnatch.filter.Filter class
     """
     return load_filter_plugins("bandersnatch_filter_plugins.release")

src/bandersnatch/package.py

@@ -147,15 +147,15 @@ def sync(self, stop_on_error=False, attempts=3):
 
     def _filter_releases(self):
         """
-        Run the release filtering plugins and remove any packages from the
-        packages_to_sync that match any filters.
+        Run the release filtering plugins and remove any releases from
+        `releases` that match any filters.
         """
         versions = list(self.releases.keys())
         for version in versions:
-            filter = False
+            filter_ = False
             for plugin in filter_release_plugins():
-                filter = filter or plugin.check_match(name=self.name, version=version)
-            if filter:
+                filter_ = filter_ or plugin.check_match(name=self.name, version=version)
+            if filter_:
                 del (self.releases[version])
 
     # TODO: async def once we go full asyncio - Have concurrency at the

src/bandersnatch/tests/test__bandersnatch_plugins__blacklist_name.py → src/bandersnatch/tests/plugins/test_blacklist_name.py

@@ -7,6 +7,7 @@
 from bandersnatch.configuration import BandersnatchConfig
 from bandersnatch.master import Master
 from bandersnatch.mirror import Mirror
+from bandersnatch.package import Package
 
 
 class TestBlacklistProject(TestCase):
@@ -191,3 +192,26 @@ def test__plugin__loads__default(self):
         plugins = bandersnatch.filter.filter_release_plugins()
         names = [plugin.name for plugin in plugins]
         self.assertIn("blacklist_release", names)
+
+    def test__filter__matches__release(self):
+        with open("test.conf", "w") as testconfig_handle:
+            testconfig_handle.write(
+                """\
+[blacklist]
+plugins =
+    blacklist_release
+packages =
+    foo==1.2.0
+"""
+            )
+        instance = BandersnatchConfig()
+        instance.config_file = "test.conf"
+        instance.load_configuration()
+
+        mirror = Mirror(".", Master(url="https://foo.bar.com"))
+        pkg = Package("foo", 1, mirror)
+        pkg.releases = {"1.2.0": {}, "1.2.1": {}}
+
+        pkg._filter_releases()
+
+        self.assertEqual(pkg.releases, {"1.2.1": {}})

src/bandersnatch/tests/plugins/test_prerelease_name.py

@@ -0,0 +1,91 @@
+import os
+import re
+from collections import defaultdict
+from tempfile import TemporaryDirectory
+from unittest import TestCase
+
+import bandersnatch.filter
+from bandersnatch.configuration import BandersnatchConfig
+from bandersnatch.master import Master
+from bandersnatch.mirror import Mirror
+from bandersnatch.package import Package
+from bandersnatch_filter_plugins import prerelease_name
+
+
+def _mock_config(contents, filename="test.conf"):
+    """
+    Creates a config file with contents and loads them into a
+    BandersnatchConfig instance.
+    """
+    with open(filename, "w") as fd:
+        fd.write(contents)
+
+    instance = BandersnatchConfig()
+    instance.config_file = filename
+    instance.load_configuration()
+    return instance
+
+
+class BasePluginTestCase(TestCase):
+
+    tempdir = None
+    cwd = None
+
+    def setUp(self):
+        self.cwd = os.getcwd()
+        self.tempdir = TemporaryDirectory()
+        bandersnatch.filter.loaded_filter_plugins = defaultdict(list)
+        os.chdir(self.tempdir.name)
+
+    def tearDown(self):
+        if self.tempdir:
+            os.chdir(self.cwd)
+            self.tempdir.cleanup()
+            self.tempdir = None
+
+
+class TestRegexReleaseFilter(BasePluginTestCase):
+
+    config_contents = """\
+[blacklist]
+plugins =
+    prerelease_release
+"""
+
+    def test_plugin_includes_predefined_patterns(self):
+        _mock_config(self.config_contents)
+
+        plugins = bandersnatch.filter.filter_release_plugins()
+
+        assert any(
+            type(plugin) == prerelease_name.PreReleaseFilter for plugin in plugins
+        )
+        plugin = next(
+            plugin
+            for plugin in plugins
+            if type(plugin) == prerelease_name.PreReleaseFilter
+        )
+        expected_patterns = [
+            re.compile(pattern_string) for pattern_string in plugin.PRERELEASE_PATTERNS
+        ]
+        assert plugin.patterns == expected_patterns
+
+    def test_plugin_check_match(self):
+        _mock_config(self.config_contents)
+
+        bandersnatch.filter.filter_release_plugins()
+
+        mirror = Mirror(".", Master(url="https://foo.bar.com"))
+        pkg = Package("foo", 1, mirror)
+        pkg.releases = {
+            "1.2.0alpha1": {},
+            "1.2.0a2": {},
+            "1.2.0beta1": {},
+            "1.2.0b2": {},
+            "1.2.0rc1": {},
+            "1.2.0": {},
+        }
+
+        pkg._filter_releases()
+
+        assert pkg.releases == {"1.2.0": {}}

src/bandersnatch/tests/plugins/test_regex_name.py

@@ -0,0 +1,122 @@
+import os
+import re
+from collections import defaultdict
+from tempfile import TemporaryDirectory
+from unittest import TestCase
+
+import bandersnatch.filter
+from bandersnatch.configuration import BandersnatchConfig
+from bandersnatch.master import Master
+from bandersnatch.mirror import Mirror
+from bandersnatch.package import Package
+from bandersnatch_filter_plugins import regex_name
+
+
+def _mock_config(contents, filename="test.conf"):
+    """
+    Creates a config file with contents and loads them into a
+    BandersnatchConfig instance.
+    """
+    with open(filename, "w") as fd:
+        fd.write(contents)
+
+    instance = BandersnatchConfig()
+    instance.config_file = filename
+    instance.load_configuration()
+    return instance
+
+
+class BasePluginTestCase(TestCase):
+
+    tempdir = None
+    cwd = None
+
+    def setUp(self):
+        self.cwd = os.getcwd()
+        self.tempdir = TemporaryDirectory()
+        bandersnatch.filter.loaded_filter_plugins = defaultdict(list)
+        os.chdir(self.tempdir.name)
+
+    def tearDown(self):
+        if self.tempdir:
+            os.chdir(self.cwd)
+            self.tempdir.cleanup()
+            self.tempdir = None
+
+
+class TestRegexReleaseFilter(BasePluginTestCase):
+
+    config_contents = """\
+[blacklist]
+plugins =
+    regex_release
+
+[filter_regex]
+releases =
+    .+rc\\d$
+    .+alpha\\d$
+"""
+
+    def test_plugin_compiles_patterns(self):
+        _mock_config(self.config_contents)
+
+        plugins = bandersnatch.filter.filter_release_plugins()
+
+        assert any(type(plugin) == regex_name.RegexReleaseFilter for plugin in plugins)
+        plugin = next(
+            plugin
+            for plugin in plugins
+            if type(plugin) == regex_name.RegexReleaseFilter
+        )
+        assert plugin.patterns == [re.compile(r".+rc\d$"), re.compile(r".+alpha\d$")]
+
+    def test_plugin_check_match(self):
+        _mock_config(self.config_contents)
+
+        bandersnatch.filter.filter_release_plugins()
+
+        mirror = Mirror(".", Master(url="https://foo.bar.com"))
+        pkg = Package("foo", 1, mirror)
+        pkg.releases = {"foo-1.2.0rc2": {}, "foo-1.2.0": {}, "foo-1.2.0alpha2": {}}
+
+        pkg._filter_releases()
+
+        assert pkg.releases == {"foo-1.2.0": {}}
+
+
+class TestRegexProjectFilter(BasePluginTestCase):
+
+    config_contents = """\
+[blacklist]
+plugins =
+    regex_project
+
+[filter_regex]
+packages =
+    .+-evil$
+    .+-neutral$
+"""
+
+    def test_plugin_compiles_patterns(self):
+        _mock_config(self.config_contents)
+
+        plugins = bandersnatch.filter.filter_project_plugins()
+
+        assert any(type(plugin) == regex_name.RegexProjectFilter for plugin in plugins)
+        plugin = next(
+            plugin
+            for plugin in plugins
+            if type(plugin) == regex_name.RegexProjectFilter
+        )
+        assert plugin.patterns == [re.compile(r".+-evil$"), re.compile(r".+-neutral$")]
+
+    def test_plugin_check_match(self):
+        _mock_config(self.config_contents)
+
+        bandersnatch.filter.filter_release_plugins()
+
+        mirror = Mirror(".", Master(url="https://foo.bar.com"))
+        mirror.packages_to_sync = {"foo-good": {}, "foo-evil": {}, "foo-neutral": {}}
+        mirror._filter_packages()
+
+        assert list(mirror.packages_to_sync.keys()) == ["foo-good"]

src/bandersnatch_filter_plugins/blacklist_name.py

@@ -125,7 +125,7 @@ def _determine_filtered_package_requirements(self):
 
     def check_match(self, **kwargs):
         """
-        Check if the package name  and version matches against a blacklisted
+        Check if the package name and version matches against a blacklisted
         package version specifier.
 
         Parameters

src/bandersnatch_filter_plugins/prerelease_name.py

@@ -0,0 +1,43 @@
+import logging
+import re
+
+from bandersnatch.filter import FilterReleasePlugin
+
+logger = logging.getLogger("bandersnatch")
+
+
+class PreReleaseFilter(FilterReleasePlugin):
+    """
+    Filters releases considered pre-releases.
+    """
+
+    name = "prerelease_release"
+    PRERELEASE_PATTERNS = (r".+rc\d$", r".+a(lpha)?\d$", r".+b(eta)?\d$")
+
+    def initialize_plugin(self):
+        """
+        Initialize the plugin reading patterns from the config.
+        """
+        self.patterns = [
+            re.compile(pattern_string) for pattern_string in self.PRERELEASE_PATTERNS
+        ]
+
+        logger.info(f"Initialized prerelease plugin with {self.patterns}")
+
+    def check_match(self, name, version):
+        """
+        Check if a release version matches any of the specificed patterns.
+
+        Parameters
+        ==========
+        name: str
+            Release name
+        version: str
+            Release version
+
+        Returns
+        =======
+        bool:
+            True if it matches, False otherwise.
+        """
+        return any(pattern.match(version) for pattern in self.patterns)