From a34663d4d96080059c68384e7c4b27029e3f43c9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 17 May 2022 14:38:37 +0200 Subject: [PATCH] fix handling of multiple iamges --- Dockerfile | 2 +- src/docker_publisher_osparc_services/cli.py | 22 ++-- .../exceptions.py | 11 ++ .../gitlab_ci_setup/commands.py | 109 +++++++++++++----- .../http_interface.py | 4 +- 5 files changed, 104 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ae5a78..d40a7f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ LABEL org.opencontainers.image.licenses="MIT" ARG REPO_NAME="https://github.com/GitHK/osparc-simcore-forked.git" ARG BRANCH_NAME="service-integration-library-additions" -ARG COMMIT_SHA="daa592ffc46d88ea61661806c63f3f9e4c04771d" +ARG COMMIT_SHA="7d701dfeeb5424cc0fc093d670a87e6ed607be82" ARG CLONE_DIR="/opsarc" ARG PYTHON_VERSION="3.8.10" ARG DEBIAN_FRONTEND=noninteractive diff --git a/src/docker_publisher_osparc_services/cli.py b/src/docker_publisher_osparc_services/cli.py index 2b77a7e..35b646e 100644 --- a/src/docker_publisher_osparc_services/cli.py +++ b/src/docker_publisher_osparc_services/cli.py @@ -5,10 +5,10 @@ from . import __version__ from .gitlab_ci_setup.commands import ( - COMMANDS_BUILD, - COMMANDS_PUSH, - COMMANDS_TEST_BASE, assemble_env_vars, + get_build_commands, + get_push_commands, + get_test_commands, validate_commands_list, ) from .gitlab_ci_setup.pipeline_config import PipelineConfig, PipelineGenerator @@ -78,28 +78,32 @@ async def run_command(config: Path) -> None: env_vars = assemble_env_vars( repo_model=repo_model, registries=cfg.registries, - image_name=image_name, tag=tag, ) - validate_commands_list(COMMANDS_BUILD, env_vars) + image_count = len(repo_model.registry.local_to_test) + + build_commands = get_build_commands(image_count) + validate_commands_list(build_commands, env_vars) # check if test stage is required test_commands = None if repo_model.ci_stage_test_script is not None: # test commands assembly and validation test_commands = ( - COMMANDS_TEST_BASE + repo_model.ci_stage_test_script + get_test_commands(image_count) + + repo_model.ci_stage_test_script ) validate_commands_list(test_commands, env_vars) # deploy stage validation - validate_commands_list(COMMANDS_PUSH, env_vars) + push_commands = get_push_commands(image_count) + validate_commands_list(push_commands, env_vars) pipeline_config = PipelineConfig( target=image_name, - build=COMMANDS_BUILD, + build=build_commands, test=test_commands, - push=COMMANDS_PUSH, + push=push_commands, ) pipeline_config.write_config() await pipeline_generator.add_pipeline_from( diff --git a/src/docker_publisher_osparc_services/exceptions.py b/src/docker_publisher_osparc_services/exceptions.py index 3ee1055..48cabb6 100644 --- a/src/docker_publisher_osparc_services/exceptions.py +++ b/src/docker_publisher_osparc_services/exceptions.py @@ -14,5 +14,16 @@ class CouldNotFindAGitlabRepositoryRepoException(BaseAppException): """not found the searched repository""" +class IncorrectImageMapping(BaseAppException): + """not found the searched repository""" + + def __init__(self, local_to_test, test_to_release) -> None: + self.local_to_test = local_to_test + self.test_to_release = test_to_release + super().__init__( + f"Could not map all images from {local_to_test=} to {test_to_release=}" + ) + + class CommandFailedException(BaseAppException): """raised if a command fails""" diff --git a/src/docker_publisher_osparc_services/gitlab_ci_setup/commands.py b/src/docker_publisher_osparc_services/gitlab_ci_setup/commands.py index d9f6ae7..a498c61 100644 --- a/src/docker_publisher_osparc_services/gitlab_ci_setup/commands.py +++ b/src/docker_publisher_osparc_services/gitlab_ci_setup/commands.py @@ -4,64 +4,109 @@ from typing import Dict, List from ..models import RegistryEndpointyModel, RepoModel +from ..exceptions import IncorrectImageMapping + +CommandList = List[str] DOCKER_LOGIN: str = ( "echo ${SCCI_TARGET_REGISTRY_PASSWORD} | " "docker login ${SCCI_TARGET_REGISTRY_ADDRESS} --username ${SCCI_TARGET_REGISTRY_USER} --password-stdin" ) -CommandList = List[str] -COMMANDS_BUILD: CommandList = [ - "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}", - "cd ${SCCI_CLONE_DIR}", - "ooil compose", - "docker-compose build", - DOCKER_LOGIN, - "docker tag ${SCCI_IMAGE_NAME}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}", - "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}", -] - -COMMANDS_TEST_BASE: CommandList = [ - "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}", - "cd ${SCCI_CLONE_DIR}", - DOCKER_LOGIN, - "docker pull ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}", - # if user defines extra commands those will be append here -] - -COMMANDS_PUSH: CommandList = [ - DOCKER_LOGIN, - "docker pull ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}", - "docker tag ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE}:${SCCI_TAG}", - "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE}:${SCCI_TAG}", -] +def get_build_commands(image_count: int) -> CommandList: + commands: CommandList = [ + "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}", + "cd ${SCCI_CLONE_DIR}", + "ooil compose", + "docker-compose build", + DOCKER_LOGIN, + ] + + for k in range(image_count): + commands.append( + "docker tag ${SCCI_IMAGE_NAME_%s}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG}" + % (k, k) + ) + commands.append( + "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG}" + % (k) + ) + + return commands + + +def get_test_commands(image_count: int) -> CommandList: + commands: CommandList = [ + "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}", + "cd ${SCCI_CLONE_DIR}", + DOCKER_LOGIN, + "docker pull ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}", + ] + + for k in range(image_count): + commands.append("docker pull ${SCCI_CI_IMAGE_NAME_%s}:${SCCI_TAG}" % (k)) + + return commands + + +def get_push_commands(image_count: int) -> CommandList: + commands: CommandList = [ + DOCKER_LOGIN, + ] + + for k in range(image_count): + commands.append( + "docker pull ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG}" + % (k) + ) + + commands.append( + "docker tag ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE_%s}:${SCCI_TAG}" + % (k, k) + ) + commands.append( + "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE_%s}:${SCCI_TAG}" + % (k) + ) + + return commands def assemble_env_vars( repo_model: RepoModel, registries: Dict[str, RegistryEndpointyModel], - image_name: str, tag: str, ) -> Dict[str, str]: clone_directory: Path = Path(TemporaryDirectory().name) registry: RegistryEndpointyModel = registries[repo_model.registry.target] - test_image = repo_model.registry.local_to_test[image_name] - release_image = repo_model.registry.test_to_release[test_image] - return { + env_vars: Dict[str, str] = { "SCCI_REPO": repo_model.escaped_repo, "SCCI_CLONE_DIR": f"{clone_directory}", - "SCCI_IMAGE_NAME": image_name, "SCCI_TAG": tag, - "SCCI_TEST_IMAGE": test_image, - "SCCI_RELEASE_IMAGE": release_image, "SCCI_TARGET_REGISTRY_ADDRESS": registry.address, "SCCI_TARGET_REGISTRY_PASSWORD": registry.password.get_secret_value(), "SCCI_TARGET_REGISTRY_USER": registry.user, } + # for multiple image builds + local_images: List[str] = list(repo_model.registry.local_to_test.keys()) + test_images: List[str] = list(repo_model.registry.local_to_test.values()) + release_images: List[str] = list(repo_model.registry.test_to_release.values()) + + for k, image_name in enumerate(local_images): + env_vars[f"SCCI_IMAGE_NAME_{k}"] = image_name + + for k, image_name in enumerate(test_images): + env_vars[f"SCCI_TEST_IMAGE_{k}"] = image_name + + for k, image_name in enumerate(release_images): + env_vars[f"SCCI_RELEASE_IMAGE_{k}"] = image_name + + return env_vars + def validate_commands_list( commands_list: CommandList, env_vars: Dict[str, str] diff --git a/src/docker_publisher_osparc_services/http_interface.py b/src/docker_publisher_osparc_services/http_interface.py index ac83da1..cabc327 100644 --- a/src/docker_publisher_osparc_services/http_interface.py +++ b/src/docker_publisher_osparc_services/http_interface.py @@ -39,7 +39,7 @@ async def github_did_last_repo_run_pass( async def _gitlab_get_project_id(repo_model: RepoModel) -> str: async with async_client() as client: parsed_url = URL(repo_model.address) - repo_name = parsed_url.path.split("/")[-1].strip(".git") + repo_name = parsed_url.path.split("/")[-1].replace(".git", "") host = parsed_url.host url = f"https://{host}/api/v4/projects?search={repo_name}" result = await client.get( @@ -55,7 +55,7 @@ async def _gitlab_get_project_id(repo_model: RepoModel) -> str: return repo["id"] message = ( - f"Searching for {repo_name} did not yield the deisired result {found_repos}" + f"Searching for {repo_name} did not yield the deisired result {found_repos} {parsed_url}" ) raise CouldNotFindAGitlabRepositoryRepoException(message)