Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tagging config to pass params easier #2234

Merged
merged 2 commits into from
Feb 22, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/docker-build-test-upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ jobs:
python3 -m tagging.apps.write_tags_file
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
--image ${{ inputs.image }}
--variant ${{ inputs.variant }}
--tags-dir /tmp/jupyter/tags/
shell: bash
Expand All @@ -97,7 +97,7 @@ jobs:
python3 -m tagging.apps.write_manifest
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
--image ${{ inputs.image }}
--variant ${{ inputs.variant }}
--hist-lines-dir /tmp/jupyter/hist_lines/
--manifests-dir /tmp/jupyter/manifests/
Expand Down Expand Up @@ -133,5 +133,5 @@ jobs:
python3 -m tests.run_tests
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
--image ${{ inputs.image }}
shell: bash
2 changes: 1 addition & 1 deletion .github/workflows/docker-merge-tags.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ jobs:
if: env.PUSH_TO_REGISTRY == 'true'
run: >
python3 -m tagging.apps.merge_tags
--short-image-name ${{ inputs.image }}
--image ${{ inputs.image }}
--variant ${{ inputs.variant }}
--tags-dir /tmp/jupyter/tags/
shell: bash
2 changes: 1 addition & 1 deletion .github/workflows/docker-tag-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ jobs:
python3 -m tagging.apps.apply_tags
--registry ${{ env.REGISTRY }}
--owner ${{ env.OWNER }}
--short-image-name ${{ inputs.image }}
--image ${{ inputs.image }}
--variant ${{ inputs.variant }}
--platform ${{ inputs.platform }}
--tags-dir /tmp/jupyter/tags/
Expand Down
10 changes: 5 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ build/%: ROOT_IMAGE?=ubuntu:24.04
build/%: PYTHON_VERSION?=3.12
build/%: ## build the latest image for a stack using the system's architecture
docker build $(DOCKER_BUILD_ARGS) --rm --force-rm \
--tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" \
--tag "$(REGISTRY)/$(OWNER)/$(notdir $@)" \
"./images/$(notdir $@)" \
--build-arg REGISTRY="$(REGISTRY)" \
--build-arg OWNER="$(OWNER)" \
Expand Down Expand Up @@ -82,21 +82,21 @@ hook/%: ## run post-build hooks for an image
python3 -m tagging.apps.write_tags_file \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--short-image-name "$(notdir $@)" \
--image "$(notdir $@)" \
--variant "$(VARIANT)" \
--tags-dir /tmp/jupyter/tags/
python3 -m tagging.apps.write_manifest \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--short-image-name "$(notdir $@)" \
--image "$(notdir $@)" \
--variant "$(VARIANT)" \
--hist-lines-dir /tmp/jupyter/hist_lines/ \
--manifests-dir /tmp/jupyter/manifests/ \
--repository "$(REPOSITORY)"
python3 -m tagging.apps.apply_tags \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--short-image-name "$(notdir $@)" \
--image "$(notdir $@)" \
--variant "$(VARIANT)" \
--platform "$(shell uname -m)" \
--tags-dir /tmp/jupyter/tags/
Expand Down Expand Up @@ -139,5 +139,5 @@ test/%: ## run tests against a stack
python3 -m tests.run_tests \
--registry "$(REGISTRY)" \
--owner "$(OWNER)" \
--short-image-name "$(notdir $@)"
--image "$(notdir $@)"
test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks
47 changes: 16 additions & 31 deletions tagging/apps/apply_tags.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,57 +2,42 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import logging
from pathlib import Path

import plumbum

from tagging.apps.common_cli_arguments import common_arguments_parser
from tagging.utils.get_platform import unify_aarch64
from tagging.utils.config import Config
from tagging.utils.get_prefix import get_file_prefix_for_platform

docker = plumbum.local["docker"]

LOGGER = logging.getLogger(__name__)


def apply_tags(
*,
registry: str,
owner: str,
short_image_name: str,
variant: str,
platform: str,
tags_dir: Path,
) -> None:
def apply_tags(config: Config) -> None:
"""
Tags <registry>/<owner>/<short_image_name>:latest with the tags reported by all taggers for this image
Tags <config.full_image()> with the tags reported by all taggers for this image
"""
LOGGER.info(f"Tagging image: {short_image_name}")
LOGGER.info(f"Tagging image: {config.image}")

file_prefix = get_file_prefix_for_platform(platform, variant)
image = f"{registry}/{owner}/{short_image_name}:latest"
filename = f"{file_prefix}-{short_image_name}.txt"
tags = (tags_dir / filename).read_text().splitlines()
file_prefix = get_file_prefix_for_platform(config.platform, config.variant)
filename = f"{file_prefix}-{config.image}.txt"
tags = (config.tags_dir / filename).read_text().splitlines()

for tag in tags:
LOGGER.info(f"Applying tag: {tag}")
docker["tag", image, tag] & plumbum.FG
docker["tag", config.full_image(), tag] & plumbum.FG


if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)

arg_parser = common_arguments_parser(
registry=True, owner=True, short_image_name=True, variant=True, tags_dir=True
config = common_arguments_parser(
registry=True,
owner=True,
image=True,
variant=True,
platform=True,
tags_dir=True,
)
arg_parser.add_argument(
"--platform",
required=True,
type=str,
choices=["x86_64", "aarch64", "arm64"],
help="Image platform",
)
args = arg_parser.parse_args()
args.platform = unify_aarch64(args.platform)

apply_tags(**vars(args))
apply_tags(config)
32 changes: 27 additions & 5 deletions tagging/apps/common_cli_arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,22 @@
import argparse
from pathlib import Path

from tagging.utils.config import Config
from tagging.utils.get_platform import unify_aarch64


def common_arguments_parser(
*,
registry: bool = False,
owner: bool = False,
short_image_name: bool = False,
image: bool = False,
variant: bool = False,
platform: bool = False,
tags_dir: bool = False,
hist_lines_dir: bool = False,
manifests_dir: bool = False,
) -> argparse.ArgumentParser:
repository: bool = False,
) -> Config:
"""Add common CLI arguments to parser"""

parser = argparse.ArgumentParser()
Expand All @@ -30,9 +35,9 @@ def common_arguments_parser(
required=True,
help="Owner of the image",
)
if short_image_name:
if image:
parser.add_argument(
"--short-image-name",
"--image",
required=True,
help="Short image name",
)
Expand All @@ -42,6 +47,14 @@ def common_arguments_parser(
required=True,
help="Variant tag prefix",
)
if platform:
parser.add_argument(
"--platform",
required=True,
type=str,
choices=["x86_64", "aarch64", "arm64"],
help="Image platform",
)
if tags_dir:
parser.add_argument(
"--tags-dir",
Expand All @@ -63,5 +76,14 @@ def common_arguments_parser(
type=Path,
help="Directory for manifests file",
)
if repository:
parser.add_argument(
"--repository",
required=True,
help="Repository name on GitHub",
)
args = parser.parse_args()
if platform:
args.platform = unify_aarch64(args.platform)

return parser
return Config(**vars(args))
25 changes: 8 additions & 17 deletions tagging/apps/merge_tags.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import logging
from pathlib import Path

import plumbum

from tagging.apps.common_cli_arguments import common_arguments_parser
from tagging.utils.config import Config
from tagging.utils.get_platform import ALL_PLATFORMS
from tagging.utils.get_prefix import get_file_prefix_for_platform

Expand All @@ -15,23 +15,18 @@
LOGGER = logging.getLogger(__name__)


def merge_tags(
*,
short_image_name: str,
variant: str,
tags_dir: Path,
) -> None:
def merge_tags(config: Config) -> None:
"""
Merge tags for x86_64 and aarch64 images when possible.
"""
LOGGER.info(f"Merging tags for image: {short_image_name}")
LOGGER.info(f"Merging tags for image: {config.image}")

all_tags: set[str] = set()

for platform in ALL_PLATFORMS:
file_prefix = get_file_prefix_for_platform(platform, variant)
filename = f"{file_prefix}-{short_image_name}.txt"
file_path = tags_dir / filename
file_prefix = get_file_prefix_for_platform(platform, config.variant)
filename = f"{file_prefix}-{config.image}.txt"
file_path = config.tags_dir / filename
if file_path.exists():
tags = file_path.read_text().splitlines()
all_tags.update(tag.replace(platform + "-", "") for tag in tags)
Expand Down Expand Up @@ -62,9 +57,5 @@ def merge_tags(
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)

arg_parser = common_arguments_parser(
short_image_name=True, variant=True, tags_dir=True
)
args = arg_parser.parse_args()

merge_tags(**vars(args))
config = common_arguments_parser(image=True, variant=True, tags_dir=True)
merge_tags(config)
Loading