Skip to content

Commit

Permalink
instead of getting tags from repo query, (#105)
Browse files Browse the repository at this point in the history 
query tags explicitly because there can be more than 500 tags in the
repository and
it can happen that additional artifacts sig/sbom/att/src are in those
first 500 tags, but image is in 500+
in that case pruner would consider artifacts orphaned and prune them

and also of course if there was 500+ tags, those wouldn't be pruned at
all

Signed-off-by: Robert Cerven <[email protected]>
  • Loading branch information
@rcerven
rcerven authored Apr 15, 2024
1 parent a5b312b commit 0f0b61c
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 72 deletions.
63 changes: 40 additions & 23 deletions config/registry_image_pruner/image_pruner/prune_images.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,42 @@
ImageRepo = Dict[str, Any]


def get_quay_repo(quay_token: str, namespace: str, name: str) -> ImageRepo:
api_url = f"{QUAY_API_URL}/repository/{namespace}/{name}"
request = Request(api_url, headers={
"Authorization": f"Bearer {quay_token}",
})
def get_quay_tags(quay_token: str, namespace: str, name: str) -> List[ImageRepo]:
next_page = None
resp: HTTPResponse
try:

all_tags = []
while True:
query_args = {"limit": 100, "onlyActiveTags": True}
if next_page is not None:
query_args["page"] = next_page

api_url = f"{QUAY_API_URL}/repository/{namespace}/{name}/tag/?{urlencode(query_args)}"
request = Request(api_url, headers={
"Authorization": f"Bearer {quay_token}",
})

with urlopen(request) as resp:
if resp.status != 200:
raise RuntimeError(resp.reason)
return json.loads(resp.read())
json_data = json.loads(resp.read())

except HTTPError as ex:
# ignore if not found
if ex.status != 404:
raise(ex)
tags = json_data.get("tags", [])
all_tags.extend(tags)

if not tags:
LOGGER.debug("No tags found.")
break

page = json_data.get("page", None)
additional = json_data.get("has_additional", False)

if additional:
next_page = page + 1
else:
return {}
break

return all_tags


def delete_image_tag(quay_token: str, namespace: str, name: str, tag: str) -> None:
Expand All @@ -61,34 +79,33 @@ def delete_image_tag(quay_token: str, namespace: str, name: str, tag: str) -> No
raise(ex)


def remove_tags(tags: Dict[str, Any], quay_token: str, namespace: str, name: str, dry_run: bool = False) -> None:
image_digests = [image["manifest_digest"] for image in tags.values()]
def remove_tags(tags: List[Dict[str, Any]], quay_token: str, namespace: str, name: str, dry_run: bool = False) -> None:
image_digests = [image["manifest_digest"] for image in tags]
tag_regex = re.compile(r"^sha256-([0-9a-f]+)(\.sbom|\.att|\.src|\.sig)$")
for tag in tags:
# attestation or sbom image
if (match := tag_regex.match(tag)) is not None:
if (match := tag_regex.match(tag["name"])) is not None:
if f"sha256:{match.group(1)}" not in image_digests:
if dry_run:
LOGGER.info("Image %s from %s/%s should be removed", tag, namespace, name)
LOGGER.info("Tag %s from %s/%s should be removed", tag["name"], namespace, name)
else:
LOGGER.info("Removing image %s from %s/%s", tag, namespace, name)
delete_image_tag(quay_token, namespace, name, tag)
LOGGER.info("Removing tag %s from %s/%s", tag["name"], namespace, name)
delete_image_tag(quay_token, namespace, name, tag["name"])
else:
LOGGER.debug("%s is not an image with suffix .att or .sbom", tag)
LOGGER.debug("%s is not an tag with suffix .att or .sbom", tag["name"])


def process_repositories(repos: List[ImageRepo], quay_token: str, dry_run: bool = False) -> None:
for repo in repos:
namespace = repo["namespace"]
name = repo["name"]
LOGGER.info("Processing repository %s: %s/%s", next(processed_repos_counter), namespace, name)
repo_info = get_quay_repo(quay_token, namespace, name)
all_tags = get_quay_tags(quay_token, namespace, name)

if not repo_info:
if not all_tags:
continue

if (tags := repo_info.get("tags")) is not None:
remove_tags(tags, quay_token, namespace, name, dry_run=dry_run)
remove_tags(all_tags, quay_token, namespace, name, dry_run=dry_run)


def fetch_image_repos(access_token: str, namespace: str) -> Iterator[List[ImageRepo]]:
Expand Down
96 changes: 47 additions & 49 deletions config/registry_image_pruner/image_pruner/test_prune_images.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def assert_quay_token_included(self, request: Request) -> None:
@patch.dict(os.environ, {"QUAY_TOKEN": QUAY_TOKEN})
@patch("sys.argv", ["prune_images", "--namespace", "sample"])
@patch("prune_images.urlopen")
@patch("prune_images.get_quay_repo")
@patch("prune_images.get_quay_tags")
def test_no_image_repo_is_fetched(self, get_quay_repo, urlopen):
response = MagicMock()
response.status = 200
Expand Down Expand Up @@ -56,10 +56,10 @@ def test_no_image_with_expected_suffixes_is_found(self, delete_image_tag, urlope
response.status = 200
# no .att or .sbom suffix here
response.read.return_value = json.dumps({
"tags": {
"latest": {"name": "latest", "manifest_digest": "sha256:03fabe17d4c5"},
"devel": {"name": "devel", "manifest_digest": "sha256:071c766795a0"},
},
"tags": [
{"name": "latest", "manifest_digest": "sha256:03fabe17d4c5"},
{"name": "devel", "manifest_digest": "sha256:071c766795a0"},
],
}).encode()
get_repo_rv.__enter__.return_value = response

Expand Down Expand Up @@ -90,7 +90,8 @@ def test_no_image_with_expected_suffixes_is_found(self, delete_image_tag, urlope

get_repo_call = urlopen.mock_calls[1]
request: Request = get_repo_call.args[0]
self.assertEqual(f"{QUAY_API_URL}/repository/sample/hello-image", request.get_full_url())
self.assertEqual(f"{QUAY_API_URL}/repository/sample/hello-image/tag/"
f"?limit=100&onlyActiveTags=True", request.get_full_url())
self.assert_make_get_request(request)
self.assert_quay_token_included(request)

Expand All @@ -113,35 +114,35 @@ def test_remove_orphan_tags_with_expected_suffixes(self, urlopen):
response.status = 200
# no .att or .sbom suffix here
response.read.return_value = json.dumps({
"tags": {
"latest": {"name": "latest", "manifest_digest": "sha256:93a8743dc130"},
"tags": [
{"name": "latest", "manifest_digest": "sha256:93a8743dc130"},
# image manifest sha256:03fabe17d4c5 does not exist
"sha256-03fabe17d4c5.sbom": {
{
"name": "sha256-03fabe17d4c5.sbom",
"manifest_digest": "sha256:e45fad41f2ff",
},
"sha256-03fabe17d4c5.att": {
{
"name": "sha256-03fabe17d4c5.att",
"manifest_digest": "sha256:e45fad41f2ff",
},
"sha256-03fabe17d4c5.src": {
{
"name": "sha256-03fabe17d4c5.src",
"manifest_digest": "sha256:f490ad41f2cc",
},
# image manifest sha256:071c766795a0 does not exist
"sha256-071c766795a0.sbom": {
{
"name": "sha256-071c766795a0.sbom",
"manifest_digest": "sha256:961207f62413",
},
"sha256-071c766795a0.att": {
{
"name": "sha256-071c766795a0.att",
"manifest_digest": "sha256:961207f62413",
},
"sha256-071c766795a0.src": {
{
"name": "sha256-071c766795a0.src",
"manifest_digest": "sha256:0ab207f62413",
},
},
],
}).encode()
get_repo_rv.__enter__.return_value = response

Expand Down Expand Up @@ -200,14 +201,11 @@ def test_remove_tag_dry_run(self, urlopen):
response = MagicMock()
response.status = 200
response.read.return_value = json.dumps({
"tags": {
"latest": {"manifest_digest": "sha256:93a8743dc130"},
"tags": [
{"name": "latest", "manifest_digest": "sha256:93a8743dc130"},
# dry run on this one
"sha256-071c766795a0.sbom": {
"name": "sha256-071c766795a0.sbom",
"manifest_digest": "sha256:961207f62413",
},
},
{"name": "sha256-071c766795a0.sbom", "manifest_digest": "sha256:961207f62413"},
],
}).encode()
get_repo_rv.__enter__.return_value = response

Expand All @@ -222,7 +220,7 @@ def test_remove_tag_dry_run(self, urlopen):
main()
dry_run_log = [
msg for msg in logs.output
if re.search(r"Image sha256-071c766795a0.sbom from [^ /]+/[^ ]+ should be removed$", msg)
if re.search(r"Tag sha256-071c766795a0.sbom from [^ /]+/[^ ]+ should be removed$", msg)
]
self.assertEqual(1, len(dry_run_log))

Expand Down Expand Up @@ -279,64 +277,64 @@ class TestRemoveTags(unittest.TestCase):

@patch("prune_images.delete_image_tag")
def test_remove_tags(self, delete_image_tag):
tags = {
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att": {
tags = [
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att",
"manifest_digest": "sha256:125c1d18ee1c3b9bde0c7810fcb0d4ffbc67e9b0c5b88bb8df9ca039bc1c9457",
},
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom": {
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom",
"manifest_digest": "sha256:351326f899759a9a7ae3ca3c1cbdadcc8012f43231c145534820a68bdf36d55b",
},
}
]

with self.assertLogs(LOGGER) as logs:
remove_tags(tags, QUAY_TOKEN, "some", "repository")
logs_output = "\n".join(logs.output)
for tag in tags:
self.assertIn(f"Removing image {tag} from some/repository", logs_output)
self.assertIn(f"Removing tag {tag['name']} from some/repository", logs_output)

self.assertEqual(len(tags), delete_image_tag.call_count)
calls = [call(QUAY_TOKEN, "some", "repository", tag) for tag in tags]
calls = [call(QUAY_TOKEN, "some", "repository", tag['name']) for tag in tags]
delete_image_tag.assert_has_calls(calls)

@patch("prune_images.delete_image_tag")
def test_remove_tags_dry_run(self, delete_image_tag):
tags = {
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att": {
tags = [
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att",
"manifest_digest": "sha256:125c1d18ee1c3b9bde0c7810fcb0d4ffbc67e9b0c5b88bb8df9ca039bc1c9457",
},
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom": {
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom",
"manifest_digest": "sha256:351326f899759a9a7ae3ca3c1cbdadcc8012f43231c145534820a68bdf36d55b",
},
}
}
]

with self.assertLogs(LOGGER) as logs:
remove_tags(tags, QUAY_TOKEN, "some", "repository", dry_run=True)
logs_output = "\n".join(logs.output)
for tag in tags:
self.assertIn(f"Image {tag} from some/repository should be removed", logs_output)
self.assertIn(f"Tag {tag['name']} from some/repository should be removed", logs_output)

delete_image_tag.assert_not_called()

@patch("prune_images.delete_image_tag")
def test_remove_tags_nothing_to_remove(self, delete_image_tag):
tags = {
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att": {
tags = [
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att",
"manifest_digest": "sha256:125c1d18ee1c3b9bde0c7810fcb0d4ffbc67e9b0c5b88bb8df9ca039bc1c9457",
},
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom": {
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom",
"manifest_digest": "sha256:351326f899759a9a7ae3ca3c1cbdadcc8012f43231c145534820a68bdf36d55b",
},
"app-image": {
{
"name": "app-image",
"manifest_digest": "sha256:502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd",
},
}
}
]

with self.assertRaisesRegex(AssertionError, expected_regex="no logs of level INFO"):
with self.assertLogs(LOGGER) as logs:
Expand All @@ -346,33 +344,33 @@ def test_remove_tags_nothing_to_remove(self, delete_image_tag):

@patch("prune_images.delete_image_tag")
def test_remove_tags_multiple_tags(self, delete_image_tag):
tags = {
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att": {
tags = [
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.att",
"manifest_digest": "sha256:125c1d18ee1c3b9bde0c7810fcb0d4ffbc67e9b0c5b88bb8df9ca039bc1c9457",
},
"sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom": {
{
"name": "sha256-502c8c35e31459e8774f88e115d50d2ad33ba0e9dfd80429bc70ed4c1fd9e0cd.sbom",
"manifest_digest": "sha256:351326f899759a9a7ae3ca3c1cbdadcc8012f43231c145534820a68bdf36d55b",
},
"sha256-5c55025c0cfc402b2a42f9d35b14a92b1ba203407d2a81aad7ea3eae1a3737d4.att": {
{
"name": "sha256-5c55025c0cfc402b2a42f9d35b14a92b1ba203407d2a81aad7ea3eae1a3737d4.att",
"manifest_digest": "sha256:5126ed26d60fffab5f82783af65b5a8e69da0820b723eea82a0eb71b0743c191",
},
"sha256-5c55025c0cfc402b2a42f9d35b14a92b1ba203407d2a81aad7ea3eae1a3737d4.sbom": {
{
"name": "sha256-5c55025c0cfc402b2a42f9d35b14a92b1ba203407d2a81aad7ea3eae1a3737d4.sbom",
"manifest_digest": "sha256:9b1f70d94117c63ee73d53688a3e4d412c1ba58d86b8e45845cce9b8dab44113",
},
}
]

with self.assertLogs(LOGGER) as logs:
remove_tags(tags, QUAY_TOKEN, "some", "repository")
logs_output = "\n".join(logs.output)
for tag in tags:
self.assertIn(f"Removing image {tag} from some/repository", logs_output)
self.assertIn(f"Removing tag {tag['name']} from some/repository", logs_output)

self.assertEqual(len(tags), delete_image_tag.call_count)
calls = [call(QUAY_TOKEN, "some", "repository", tag) for tag in tags]
calls = [call(QUAY_TOKEN, "some", "repository", tag["name"]) for tag in tags]
delete_image_tag.assert_has_calls(calls)


Expand Down

0 comments on commit 0f0b61c

Please sign in to comment.