diff --git a/README.md b/README.md index fd706e0..8aceeac 100644 --- a/README.md +++ b/README.md @@ -165,14 +165,14 @@ garbage collection. So, make sure you run something like below cd [path-where-your-docker-compose.yml] docker-compose stop registry docker-compose run --rm \ - registry bin/registry garbage-collect \ + registry bin/registry garbage-collect --delete-untagged \ /etc/docker/registry/config.yml docker-compose up -d registry ``` or (if you are not using docker-compose): ``` docker stop registry:2 - docker run --rm registry:2 bin/registry garbage-collect \ + docker run --rm registry:2 bin/registry garbage-collect --delete-untagged \ /etc/docker/registry/config.yml docker start registry:2 ``` diff --git a/registry.py b/registry.py index a365560..6214786 100755 --- a/registry.py +++ b/registry.py @@ -1,5 +1,11 @@ #!/usr/bin/env python +###### +# github repository: https://github.com/andrey-pohilko/registry-cli +# +# please read more details about the script, usage options and license info there +###### + import requests import ast from requests.packages.urllib3.exceptions import InsecureRequestWarning @@ -11,10 +17,11 @@ import os import argparse import www_authenticate -import _strptime from datetime import timedelta, datetime as dt from getpass import getpass from multiprocessing.pool import ThreadPool +from dateutil.parser import parse +from dateutil.tz import tzutc # this is a registry manipulator, can do following: # - list all images (including layers) @@ -546,6 +553,12 @@ def parse_args(args=None): default='POST', metavar="POST|GET" ) + parser.add_argument( + '--order-by-date', + help=('Orders images by date instead of by tag name.' + 'Useful if your tag names are not in a fixed order.'), + action='store_true' + ) return parser.parse_args(args) @@ -636,7 +649,7 @@ def delete_tags_by_age(registry, image_name, dry_run, hours, tags_to_keep): print("timestamp not found") continue - if dt.strptime(image_age[:-4], "%Y-%m-%dT%H:%M:%S.%f") < dt.now() - timedelta(hours=int(hours)): + if parse(image_age).astimezone(tzutc()) < dt.now(tzutc()) - timedelta(hours=int(hours)): print("will be deleted tag: {0} timestamp: {1}".format( tag, image_age)) tags_to_delete.append(tag) @@ -655,7 +668,7 @@ def newer(tag): if image_age == []: print("timestamp not found") return None - if dt.strptime(image_age[:-4], "%Y-%m-%dT%H:%M:%S.%f") >= dt.now() - timedelta(hours=int(hours)): + if parse(image_age).astimezone(tzutc()) >= dt.now(tzutc()) - timedelta(hours=int(hours)): print("Keeping tag: {0} timestamp: {1}".format( tag, image_age)) return tag @@ -672,6 +685,29 @@ def newer(tag): return result +def get_datetime_tags(registry, image_name, tags_list): + def newer(tag): + image_config = registry.get_tag_config(image_name, tag) + if image_config == []: + print("tag not found") + return None + image_age = registry.get_image_age(image_name, image_config) + if image_age == []: + print("timestamp not found") + return None + return { + "tag": tag, + "datetime": parse(image_age).astimezone(tzutc()) + } + + print('---------------------------------') + p = ThreadPool(4) + result = list(x for x in p.map(newer, tags_list) if x) + p.close() + p.join() + return result + + def keep_images_like(image_list, regexp_list): if image_list is None or regexp_list is None: return [] @@ -685,6 +721,18 @@ def keep_images_like(image_list, regexp_list): return result +def get_ordered_tags(registry, image_name, tags_list, order_by_date=False): + if order_by_date: + tags_date = get_datetime_tags(registry, image_name, tags_list) + sorted_tags_by_date = sorted( + tags_date, + key=lambda x: x["datetime"] + ) + return [x["tag"] for x in sorted_tags_by_date] + + return sorted(tags_list, key=natural_keys) + + def main_loop(args): global DEBUG @@ -749,7 +797,10 @@ def main_loop(args): print(" no tags!") continue - tags_list = get_tags(all_tags_list, image_name, args.tags_like) + if args.order_by_date: + tags_list = get_ordered_tags(registry, image_name, all_tags_list, args.order_by_date) + else: + tags_list = get_tags(all_tags_list, image_name, args.tags_like) # print(tags and optionally layers for tag in tags_list: @@ -763,9 +814,11 @@ def main_loop(args): print(" layer: {0}".format( layer['blobSum'])) - # add tags to "tags_to_keep" list, if we have regexp "tags_to_keep" - # entries or a number of hours for "keep_by_hours": + # add tags to "tags_to_keep" list if we have regexp "tags_to_keep" + # entries, a number of hours for "keep_by_hours" or if the user + # explicitly specified tags to always keep. keep_tags = [] + keep_tags.extend(args.keep_tags) if args.keep_tags_like: keep_tags.extend(get_tags_like(args.keep_tags_like, tags_list)) if args.keep_by_hours: @@ -778,8 +831,8 @@ def main_loop(args): if args.delete_all: tags_list_to_delete = list(tags_list) else: - tags_list_to_delete = sorted(tags_list, key=natural_keys)[ - :-keep_last_versions] + ordered_tags_list = get_ordered_tags(registry, image_name, tags_list, args.order_by_date) + tags_list_to_delete = ordered_tags_list[:-keep_last_versions] # A manifest might be shared between different tags. Explicitly add those # tags that we want to preserve to the keep_tags list, to prevent @@ -788,13 +841,13 @@ def main_loop(args): tag for tag in tags_list if tag not in tags_list_to_delete] keep_tags.extend(tags_list_to_keep) + keep_tags.sort() # Make order deterministic for testing delete_tags( registry, image_name, args.dry_run, tags_list_to_delete, keep_tags) # delete tags by age in hours if args.delete_by_hours: - keep_tags.extend(args.keep_tags) delete_tags_by_age(registry, image_name, args.dry_run, args.delete_by_hours, keep_tags) diff --git a/requirements-build.txt b/requirements-build.txt index 749d9d5..e3946ce 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1,6 +1,7 @@ certifi==2017.7.27.1 chardet==3.0.4 idna>=2.5 +python-dateutil==2.8.0 requests>=2.20.0 urllib3>=1.23 www-authenticate==0.9.2 \ No newline at end of file diff --git a/requirements-ci.txt b/requirements-ci.txt index e5c72c3..59a60bb 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -2,8 +2,8 @@ mock coverage certifi chardet +python-dateutil==2.8.0 idna>=2.5 requests>=2.20.0 urllib3>=1.23 www-authenticate - diff --git a/test.py b/test.py index 7f616f4..861b318 100644 --- a/test.py +++ b/test.py @@ -1,7 +1,12 @@ import unittest + +from datetime import datetime + +from dateutil.tz import tzutc, tzoffset + from registry import Registry, Requests, get_tags, parse_args, \ delete_tags, delete_tags_by_age, get_error_explanation, get_newer_tags, \ - keep_images_like, main_loop + keep_images_like, main_loop, get_datetime_tags, get_ordered_tags from mock import MagicMock, patch import requests @@ -659,6 +664,18 @@ def test_delete_tags_by_age_no_keep(self, delete_tags_patched): delete_tags_patched.assert_called_with( self.registry, "imagename", False, ["image"], []) + @patch('registry.delete_tags') + def test_delete_tags_by_age_no_keep_with_non_utc_value(self, delete_tags_patched): + self.registry.get_image_age.return_value = "2017-12-27T12:47:33.511765448+02:00" + delete_tags_by_age(self.registry, "imagename", False, 24, []) + self.list_tags_mock.assert_called_with( + "imagename" + ) + self.list_tags_mock.assert_called_with("imagename") + self.get_tag_config_mock.assert_called_with("imagename", "image") + delete_tags_patched.assert_called_with( + self.registry, "imagename", False, ["image"], []) + @patch('registry.delete_tags') def test_delete_tags_by_age_keep_tags(self, delete_tags_patched): delete_tags_by_age(self.registry, "imagename", False, 24, ["latest"]) @@ -714,6 +731,86 @@ def test_keep_tags_by_age_keep(self): ["latest"] ) + def test_keep_tags_by_age_no_keep_non_utc_datetime(self): + self.registry.get_image_age.return_value = "2017-12-27T12:47:33.511765448+02:00" + self.assertEqual( + get_newer_tags(self.registry, "imagename", 23, ["latest"]), + [] + ) + + +class TestGetDatetimeTags(unittest.TestCase): + + def setUp(self): + self.registry = Registry() + self.registry.http = MockRequests() + + self.get_tag_config_mock = MagicMock(return_value={'mediaType': 'application/vnd.docker.container.image.v1+json', 'size': 12953, + 'digest': 'sha256:8d71dfbf239c0015ad66993d55d3954cee2d52d86f829fdff9ccfb9f23b75aa8'}) + self.registry.get_tag_config = self.get_tag_config_mock + self.get_image_age_mock = MagicMock( + return_value="2017-12-27T12:47:33.511765448Z") + self.registry.get_image_age = self.get_image_age_mock + self.list_tags_mock = MagicMock(return_value=["image"]) + self.registry.list_tags = self.list_tags_mock + self.get_tag_digest_mock = MagicMock() + self.registry.get_tag_digest = self.get_tag_digest_mock + self.registry.http = MockRequests() + self.registry.hostname = "http://testdomain.com" + self.registry.http.reset_return_value(200, "MOCK_DIGEST") + + def test_get_datetime_tags(self): + self.assertEqual( + get_datetime_tags(self.registry, "imagename", ["latest"]), + [{"tag": "latest", "datetime": datetime(2017, 12, 27, 12, 47, 33, 511765, tzinfo=tzutc())}] + ) + + def test_get_non_utc_datetime_tags(self): + self.registry.get_image_age.return_value = "2019-07-18T16:33:15.864962122+02:00" + self.assertEqual( + get_datetime_tags(self.registry, "imagename", ["latest"]), + [{"tag": "latest", "datetime": datetime(2019, 7, 18, 16, 33, 15, 864962, tzinfo=tzoffset(None, 7200))}] + ) + + +class TestGetOrderedTags(unittest.TestCase): + def setUp(self): + self.tags = ["e61d48b", "ff24a83", "ddd514c", "f4ba381", "9d5fab2"] + + def test_tags_are_ordered_by_name_by_default(self): + tags = ["v1", "v10", "v2"] + ordered_tags = get_ordered_tags(registry=None, image_name=None, tags_list=tags) + self.assertEqual(ordered_tags, ["v1", "v2", "v10"]) + + @patch('registry.get_datetime_tags') + def test_tags_are_ordered_ascending_by_date_if_the_option_is_given(self, get_datetime_tags_patched): + tags = ["e61d48b", "ff24a83", "ddd514c", "f4ba381", "9d5fab2"] + get_datetime_tags_patched.return_value = [ + { + "tag": "e61d48b", + "datetime": datetime(2025, 1, 1, tzinfo=tzutc()) + }, + { + "tag": "ff24a83", + "datetime": datetime(2024, 1, 1, tzinfo=tzutc()) + }, + { + "tag": "ddd514c", + "datetime": datetime(2023, 1, 1, tzinfo=tzutc()) + }, + { + "tag": "f4ba381", + "datetime": datetime(2022, 1, 1, tzinfo=tzutc()) + }, + { + "tag": "9d5fab2", + "datetime": datetime(2021, 1, 1, tzinfo=tzutc()) + } + ] + ordered_tags = get_ordered_tags(registry="registry", image_name="image", tags_list=tags, order_by_date=True) + get_datetime_tags_patched.assert_called_once_with("registry", "image", tags) + self.assertEqual(ordered_tags, ["9d5fab2", "f4ba381", "ddd514c", "ff24a83", "e61d48b"]) + class TestKeepImagesLike(unittest.TestCase): @@ -756,6 +853,32 @@ def test_main_calls(self, keep_images_like_patched, get_auth_schemes_patched): keep_images_like_patched.assert_not_called() +class TestKeepTags(unittest.TestCase): + @staticmethod + def create_mock_registry(host, login, no_validate_ssl, digest_method="HEAD"): + r = Registry._create(host, login, no_validate_ssl, digest_method) + r.http = MockRequests() + r.list_images = MagicMock(return_value=['a']) + r.list_tags = MagicMock(return_value=['1', '2', '3', '4', '5', '6', '7']) + return r + + # We store the actual mock registry here so we can later compare it in + # `assert_called_with()`. + mock_registry = create_mock_registry.__func__('localhost:8989', None, True) + + def return_mock_registry(self, host, login, no_validate_ssl, digest_method="HEAD"): + return TestKeepTags.mock_registry + + @patch('registry.Registry.create', return_mock_registry) + @patch('registry.get_auth_schemes') # called in main_loop, turn to noop + @patch('registry.delete_tags') + def test_keep_tags(self, delete_tags_patched, get_auth_schemes_patched): + # Check if delete_tags is called from main_loop (directly or indirectly + # through delete_tags_by_age) with `keep_tags` set to the tags we want to keep. + main_loop(parse_args(('--delete', '--num', '5', '-r', 'localhost:8989', '--keep-tags', '1'))) + delete_tags_patched.assert_called_with(TestKeepTags.mock_registry, 'a', False, ['1', '2'], ['1', '3', '4', '5', '6', '7']) + + class TestArgParser(unittest.TestCase): def test_no_args(self): @@ -776,13 +899,15 @@ def test_all_args(self): "--layers", "--delete-by-hours", "24", "--keep-by-hours", "24", - "--digest-method", "GET"] + "--digest-method", "GET", + "--order-by-date"] args = parse_args(args_list) self.assertTrue(args.delete) self.assertTrue(args.layers) self.assertTrue(args.no_validate_ssl) self.assertTrue(args.delete_all) self.assertTrue(args.layers) + self.assertTrue(args.order_by_date) self.assertEqual(args.image, ["imagename1", "imagename2"]) self.assertEqual(args.num, "15") self.assertEqual(args.login, "loginstring") @@ -798,6 +923,7 @@ def test_default_args(self): "-l", "loginstring"] args = parse_args(args_list) self.assertEqual(args.digest_method, "HEAD") + self.assertFalse(args.order_by_date) if __name__ == '__main__':