Upstream 20220116

README.md

@@ -165,14 +165,14 @@ garbage collection. So, make sure you run something like below
   cd [path-where-your-docker-compose.yml]
   docker-compose stop registry
   docker-compose run --rm \
-       registry bin/registry garbage-collect \
+       registry bin/registry garbage-collect --delete-untagged \
        /etc/docker/registry/config.yml
   docker-compose up -d registry
 ```
 or (if you are not using docker-compose):
 ```
   docker stop registry:2
-  docker run --rm registry:2 bin/registry garbage-collect \
+  docker run --rm registry:2 bin/registry garbage-collect --delete-untagged \
        /etc/docker/registry/config.yml
   docker start registry:2
 ```

registry.py

@@ -1,5 +1,11 @@
 #!/usr/bin/env python
 
+######
+# github repository: https://github.com/andrey-pohilko/registry-cli
+# 
+# please read more details about the script, usage options and license info there
+######
+
 import requests
 import ast
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
@@ -11,10 +17,11 @@
 import os
 import argparse
 import www_authenticate
-import _strptime
 from datetime import timedelta, datetime as dt
 from getpass import getpass
 from multiprocessing.pool import ThreadPool
+from dateutil.parser import parse
+from dateutil.tz import tzutc
 
 # this is a registry manipulator, can do following:
 # - list all images (including layers)
@@ -546,6 +553,12 @@ def parse_args(args=None):
          default='POST',
          metavar="POST|GET"
     )
+    parser.add_argument(
+        '--order-by-date',
+        help=('Orders images by date instead of by tag name.'
+              'Useful if your tag names are not in a fixed order.'),
+        action='store_true'
+    )
     return parser.parse_args(args)
 
 
@@ -636,7 +649,7 @@ def delete_tags_by_age(registry, image_name, dry_run, hours, tags_to_keep):
             print("timestamp not found")
             continue
 
-        if dt.strptime(image_age[:-4], "%Y-%m-%dT%H:%M:%S.%f") < dt.now() - timedelta(hours=int(hours)):
+        if parse(image_age).astimezone(tzutc()) < dt.now(tzutc()) - timedelta(hours=int(hours)):
             print("will be deleted tag: {0} timestamp: {1}".format(
                 tag, image_age))
             tags_to_delete.append(tag)
@@ -655,7 +668,7 @@ def newer(tag):
         if image_age == []:
             print("timestamp not found")
             return None
-        if dt.strptime(image_age[:-4], "%Y-%m-%dT%H:%M:%S.%f") >= dt.now() - timedelta(hours=int(hours)):
+        if parse(image_age).astimezone(tzutc()) >= dt.now(tzutc()) - timedelta(hours=int(hours)):
             print("Keeping tag: {0} timestamp: {1}".format(
                 tag, image_age))
             return tag
@@ -672,6 +685,29 @@ def newer(tag):
     return result
 
 
+def get_datetime_tags(registry, image_name, tags_list):
+    def newer(tag):
+        image_config = registry.get_tag_config(image_name, tag)
+        if image_config == []:
+            print("tag not found")
+            return None
+        image_age = registry.get_image_age(image_name, image_config)
+        if image_age == []:
+            print("timestamp not found")
+            return None
+        return {
+            "tag": tag,
+            "datetime": parse(image_age).astimezone(tzutc())
+        }
+
+    print('---------------------------------')
+    p = ThreadPool(4)
+    result = list(x for x in p.map(newer, tags_list) if x)
+    p.close()
+    p.join()
+    return result
+
+
 def keep_images_like(image_list, regexp_list):
     if image_list is None or regexp_list is None:
         return []
@@ -685,6 +721,18 @@ def keep_images_like(image_list, regexp_list):
     return result
 
 
+def get_ordered_tags(registry, image_name, tags_list, order_by_date=False):
+    if order_by_date:
+        tags_date = get_datetime_tags(registry, image_name, tags_list)
+        sorted_tags_by_date = sorted(
+            tags_date,
+            key=lambda x: x["datetime"]
+        )
+        return [x["tag"] for x in sorted_tags_by_date]
+
+    return sorted(tags_list, key=natural_keys)
+
+
 def main_loop(args):
     global DEBUG
 
@@ -749,7 +797,10 @@ def main_loop(args):
             print("  no tags!")
             continue
 
-        tags_list = get_tags(all_tags_list, image_name, args.tags_like)
+        if args.order_by_date:
+            tags_list = get_ordered_tags(registry, image_name, all_tags_list, args.order_by_date)
+        else:
+            tags_list = get_tags(all_tags_list, image_name, args.tags_like)
 
         # print(tags and optionally layers
         for tag in tags_list:
@@ -763,9 +814,11 @@ def main_loop(args):
                         print("    layer: {0}".format(
                             layer['blobSum']))
 
-        # add tags to "tags_to_keep" list, if we have regexp "tags_to_keep"
-        # entries or a number of hours for "keep_by_hours":
+        # add tags to "tags_to_keep" list if we have regexp "tags_to_keep"
+        # entries, a number of hours for "keep_by_hours" or if the user
+        # explicitly specified tags to always keep.
         keep_tags = []
+        keep_tags.extend(args.keep_tags)
         if args.keep_tags_like:
             keep_tags.extend(get_tags_like(args.keep_tags_like, tags_list))
         if args.keep_by_hours:
@@ -778,8 +831,8 @@ def main_loop(args):
             if args.delete_all:
                 tags_list_to_delete = list(tags_list)
             else:
-                tags_list_to_delete = sorted(tags_list, key=natural_keys)[
-                    :-keep_last_versions]
+                ordered_tags_list = get_ordered_tags(registry, image_name, tags_list, args.order_by_date)
+                tags_list_to_delete = ordered_tags_list[:-keep_last_versions]
 
                 # A manifest might be shared between different tags. Explicitly add those
                 # tags that we want to preserve to the keep_tags list, to prevent
@@ -788,13 +841,13 @@ def main_loop(args):
                     tag for tag in tags_list if tag not in tags_list_to_delete]
                 keep_tags.extend(tags_list_to_keep)
 
+            keep_tags.sort() # Make order deterministic for testing
             delete_tags(
                 registry, image_name, args.dry_run,
                 tags_list_to_delete, keep_tags)
 
         # delete tags by age in hours
         if args.delete_by_hours:
-            keep_tags.extend(args.keep_tags)
             delete_tags_by_age(registry, image_name, args.dry_run,
                                args.delete_by_hours, keep_tags)
 

requirements-build.txt

@@ -1,6 +1,7 @@
 certifi==2017.7.27.1
 chardet==3.0.4
 idna>=2.5
+python-dateutil==2.8.0
 requests>=2.20.0
 urllib3>=1.23
 www-authenticate==0.9.2

requirements-ci.txt

@@ -2,8 +2,8 @@ mock
 coverage
 certifi
 chardet
+python-dateutil==2.8.0
 idna>=2.5
 requests>=2.20.0
 urllib3>=1.23
 www-authenticate
-

test.py

@@ -1,7 +1,12 @@
 import unittest
+
+from datetime import datetime
+
+from dateutil.tz import tzutc, tzoffset
+
 from registry import Registry, Requests, get_tags, parse_args, \
     delete_tags, delete_tags_by_age, get_error_explanation, get_newer_tags, \
-    keep_images_like, main_loop
+    keep_images_like, main_loop, get_datetime_tags, get_ordered_tags
 from mock import MagicMock, patch
 import requests
 
@@ -659,6 +664,18 @@ def test_delete_tags_by_age_no_keep(self, delete_tags_patched):
         delete_tags_patched.assert_called_with(
             self.registry, "imagename", False, ["image"], [])
 
+    @patch('registry.delete_tags')
+    def test_delete_tags_by_age_no_keep_with_non_utc_value(self, delete_tags_patched):
+        self.registry.get_image_age.return_value = "2017-12-27T12:47:33.511765448+02:00"
+        delete_tags_by_age(self.registry, "imagename", False, 24, [])
+        self.list_tags_mock.assert_called_with(
+            "imagename"
+        )
+        self.list_tags_mock.assert_called_with("imagename")
+        self.get_tag_config_mock.assert_called_with("imagename", "image")
+        delete_tags_patched.assert_called_with(
+            self.registry, "imagename", False, ["image"], [])
+
     @patch('registry.delete_tags')
     def test_delete_tags_by_age_keep_tags(self, delete_tags_patched):
         delete_tags_by_age(self.registry, "imagename", False, 24, ["latest"])
@@ -714,6 +731,86 @@ def test_keep_tags_by_age_keep(self):
                 ["latest"]
             )
 
+        def test_keep_tags_by_age_no_keep_non_utc_datetime(self):
+            self.registry.get_image_age.return_value = "2017-12-27T12:47:33.511765448+02:00"
+            self.assertEqual(
+                get_newer_tags(self.registry, "imagename", 23, ["latest"]),
+                []
+            )
+
+
+class TestGetDatetimeTags(unittest.TestCase):
+
+    def setUp(self):
+        self.registry = Registry()
+        self.registry.http = MockRequests()
+
+        self.get_tag_config_mock = MagicMock(return_value={'mediaType': 'application/vnd.docker.container.image.v1+json', 'size': 12953,
+                                                           'digest': 'sha256:8d71dfbf239c0015ad66993d55d3954cee2d52d86f829fdff9ccfb9f23b75aa8'})
+        self.registry.get_tag_config = self.get_tag_config_mock
+        self.get_image_age_mock = MagicMock(
+            return_value="2017-12-27T12:47:33.511765448Z")
+        self.registry.get_image_age = self.get_image_age_mock
+        self.list_tags_mock = MagicMock(return_value=["image"])
+        self.registry.list_tags = self.list_tags_mock
+        self.get_tag_digest_mock = MagicMock()
+        self.registry.get_tag_digest = self.get_tag_digest_mock
+        self.registry.http = MockRequests()
+        self.registry.hostname = "http://testdomain.com"
+        self.registry.http.reset_return_value(200, "MOCK_DIGEST")
+
+    def test_get_datetime_tags(self):
+        self.assertEqual(
+            get_datetime_tags(self.registry, "imagename", ["latest"]),
+            [{"tag": "latest", "datetime": datetime(2017, 12, 27, 12, 47, 33, 511765, tzinfo=tzutc())}]
+        )
+
+    def test_get_non_utc_datetime_tags(self):
+        self.registry.get_image_age.return_value = "2019-07-18T16:33:15.864962122+02:00"
+        self.assertEqual(
+            get_datetime_tags(self.registry, "imagename", ["latest"]),
+            [{"tag": "latest", "datetime": datetime(2019, 7, 18, 16, 33, 15, 864962, tzinfo=tzoffset(None, 7200))}]
+        )
+
+
+class TestGetOrderedTags(unittest.TestCase):
+    def setUp(self):
+        self.tags = ["e61d48b", "ff24a83", "ddd514c", "f4ba381", "9d5fab2"]
+
+    def test_tags_are_ordered_by_name_by_default(self):
+        tags = ["v1", "v10", "v2"]
+        ordered_tags = get_ordered_tags(registry=None, image_name=None, tags_list=tags)
+        self.assertEqual(ordered_tags, ["v1", "v2", "v10"])
+
+    @patch('registry.get_datetime_tags')
+    def test_tags_are_ordered_ascending_by_date_if_the_option_is_given(self, get_datetime_tags_patched):
+        tags = ["e61d48b", "ff24a83", "ddd514c", "f4ba381", "9d5fab2"]
+        get_datetime_tags_patched.return_value = [
+            {
+                "tag": "e61d48b",
+                "datetime": datetime(2025, 1, 1, tzinfo=tzutc())
+            },
+            {
+                "tag": "ff24a83",
+                "datetime": datetime(2024, 1, 1, tzinfo=tzutc())
+            },
+            {
+                "tag": "ddd514c",
+                "datetime": datetime(2023, 1, 1, tzinfo=tzutc())
+            },
+            {
+                "tag": "f4ba381",
+                "datetime": datetime(2022, 1, 1, tzinfo=tzutc())
+            },
+            {
+                "tag": "9d5fab2",
+                "datetime": datetime(2021, 1, 1, tzinfo=tzutc())
+            }
+        ]
+        ordered_tags = get_ordered_tags(registry="registry", image_name="image", tags_list=tags, order_by_date=True)
+        get_datetime_tags_patched.assert_called_once_with("registry", "image", tags)
+        self.assertEqual(ordered_tags, ["9d5fab2", "f4ba381", "ddd514c", "ff24a83", "e61d48b"])
+
 
 class TestKeepImagesLike(unittest.TestCase):
 
@@ -756,6 +853,32 @@ def test_main_calls(self, keep_images_like_patched, get_auth_schemes_patched):
         keep_images_like_patched.assert_not_called()
 
 
+class TestKeepTags(unittest.TestCase):
+    @staticmethod
+    def create_mock_registry(host, login, no_validate_ssl, digest_method="HEAD"):
+        r = Registry._create(host, login, no_validate_ssl, digest_method)
+        r.http = MockRequests()
+        r.list_images = MagicMock(return_value=['a'])
+        r.list_tags = MagicMock(return_value=['1', '2', '3', '4', '5', '6', '7'])
+        return r
+
+    # We store the actual mock registry here so we can later compare it in
+    # `assert_called_with()`.
+    mock_registry = create_mock_registry.__func__('localhost:8989', None, True)
+
+    def return_mock_registry(self, host, login, no_validate_ssl, digest_method="HEAD"):
+        return TestKeepTags.mock_registry
+
+    @patch('registry.Registry.create', return_mock_registry)
+    @patch('registry.get_auth_schemes')  # called in main_loop, turn to noop
+    @patch('registry.delete_tags')
+    def test_keep_tags(self, delete_tags_patched, get_auth_schemes_patched):
+        # Check if delete_tags is called from main_loop (directly or indirectly
+        # through delete_tags_by_age) with `keep_tags` set to the tags we want to keep.
+        main_loop(parse_args(('--delete', '--num', '5', '-r', 'localhost:8989', '--keep-tags', '1')))
+        delete_tags_patched.assert_called_with(TestKeepTags.mock_registry, 'a', False, ['1', '2'], ['1', '3', '4', '5', '6', '7'])
+
+
 class TestArgParser(unittest.TestCase):
 
     def test_no_args(self):
@@ -776,13 +899,15 @@ def test_all_args(self):
                      "--layers",
                      "--delete-by-hours", "24",
                      "--keep-by-hours", "24",
-                     "--digest-method", "GET"]
+                     "--digest-method", "GET",
+                     "--order-by-date"]
         args = parse_args(args_list)
         self.assertTrue(args.delete)
         self.assertTrue(args.layers)
         self.assertTrue(args.no_validate_ssl)
         self.assertTrue(args.delete_all)
         self.assertTrue(args.layers)
+        self.assertTrue(args.order_by_date)
         self.assertEqual(args.image, ["imagename1", "imagename2"])
         self.assertEqual(args.num, "15")
         self.assertEqual(args.login, "loginstring")
@@ -798,6 +923,7 @@ def test_default_args(self):
                      "-l", "loginstring"]
         args = parse_args(args_list)
         self.assertEqual(args.digest_method, "HEAD")
+        self.assertFalse(args.order_by_date)
 
 
 if __name__ == '__main__':