fix handling of multiple iamges

Dockerfile

@@ -7,7 +7,7 @@ LABEL org.opencontainers.image.licenses="MIT"
 
 ARG REPO_NAME="https://github.com/GitHK/osparc-simcore-forked.git"
 ARG BRANCH_NAME="service-integration-library-additions"
-ARG COMMIT_SHA="daa592ffc46d88ea61661806c63f3f9e4c04771d"
+ARG COMMIT_SHA="7d701dfeeb5424cc0fc093d670a87e6ed607be82"
 ARG CLONE_DIR="/opsarc"
 ARG PYTHON_VERSION="3.8.10"
 ARG DEBIAN_FRONTEND=noninteractive

src/docker_publisher_osparc_services/cli.py

@@ -5,10 +5,10 @@
 
 from . import __version__
 from .gitlab_ci_setup.commands import (
-    COMMANDS_BUILD,
-    COMMANDS_PUSH,
-    COMMANDS_TEST_BASE,
     assemble_env_vars,
+    get_build_commands,
+    get_push_commands,
+    get_test_commands,
     validate_commands_list,
 )
 from .gitlab_ci_setup.pipeline_config import PipelineConfig, PipelineGenerator
@@ -78,28 +78,32 @@ async def run_command(config: Path) -> None:
                     env_vars = assemble_env_vars(
                         repo_model=repo_model,
                         registries=cfg.registries,
-                        image_name=image_name,
                         tag=tag,
                     )
-                    validate_commands_list(COMMANDS_BUILD, env_vars)
+                    image_count = len(repo_model.registry.local_to_test)
+
+                    build_commands = get_build_commands(image_count)
+                    validate_commands_list(build_commands, env_vars)
 
                     # check if test stage is required
                     test_commands = None
                     if repo_model.ci_stage_test_script is not None:
                         # test commands assembly and validation
                         test_commands = (
-                            COMMANDS_TEST_BASE + repo_model.ci_stage_test_script
+                            get_test_commands(image_count)
+                            + repo_model.ci_stage_test_script
                         )
                         validate_commands_list(test_commands, env_vars)
 
                     # deploy stage validation
-                    validate_commands_list(COMMANDS_PUSH, env_vars)
+                    push_commands = get_push_commands(image_count)
+                    validate_commands_list(push_commands, env_vars)
 
                     pipeline_config = PipelineConfig(
                         target=image_name,
-                        build=COMMANDS_BUILD,
+                        build=build_commands,
                         test=test_commands,
-                        push=COMMANDS_PUSH,
+                        push=push_commands,
                     )
                     pipeline_config.write_config()
                     await pipeline_generator.add_pipeline_from(

src/docker_publisher_osparc_services/exceptions.py

@@ -14,5 +14,16 @@ class CouldNotFindAGitlabRepositoryRepoException(BaseAppException):
     """not found the searched repository"""
 
 
+class IncorrectImageMapping(BaseAppException):
+    """not found the searched repository"""
+
+    def __init__(self, local_to_test, test_to_release) -> None:
+        self.local_to_test = local_to_test
+        self.test_to_release = test_to_release
+        super().__init__(
+            f"Could not map all images from {local_to_test=} to {test_to_release=}"
+        )
+
+
 class CommandFailedException(BaseAppException):
     """raised if a command fails"""

src/docker_publisher_osparc_services/gitlab_ci_setup/commands.py

@@ -4,64 +4,109 @@
 from typing import Dict, List
 
 from ..models import RegistryEndpointyModel, RepoModel
+from ..exceptions import IncorrectImageMapping
+
+CommandList = List[str]
 
 DOCKER_LOGIN: str = (
     "echo ${SCCI_TARGET_REGISTRY_PASSWORD} | "
     "docker login ${SCCI_TARGET_REGISTRY_ADDRESS} --username ${SCCI_TARGET_REGISTRY_USER} --password-stdin"
 )
 
-CommandList = List[str]
 
-COMMANDS_BUILD: CommandList = [
-    "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}",
-    "cd ${SCCI_CLONE_DIR}",
-    "ooil compose",
-    "docker-compose build",
-    DOCKER_LOGIN,
-    "docker tag ${SCCI_IMAGE_NAME}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}",
-    "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}",
-]
-
-COMMANDS_TEST_BASE: CommandList = [
-    "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}",
-    "cd ${SCCI_CLONE_DIR}",
-    DOCKER_LOGIN,
-    "docker pull ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}",
-    # if user defines extra commands those will be append here
-]
-
-COMMANDS_PUSH: CommandList = [
-    DOCKER_LOGIN,
-    "docker pull ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}",
-    "docker tag ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE}:${SCCI_TAG}",
-    "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE}:${SCCI_TAG}",
-]
+def get_build_commands(image_count: int) -> CommandList:
+    commands: CommandList = [
+        "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}",
+        "cd ${SCCI_CLONE_DIR}",
+        "ooil compose",
+        "docker-compose build",
+        DOCKER_LOGIN,
+    ]
+
+    for k in range(image_count):
+        commands.append(
+            "docker tag ${SCCI_IMAGE_NAME_%s}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG}"
+            % (k, k)
+        )
+        commands.append(
+            "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG}"
+            % (k)
+        )
+
+    return commands
+
+
+def get_test_commands(image_count: int) -> CommandList:
+    commands: CommandList = [
+        "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}",
+        "cd ${SCCI_CLONE_DIR}",
+        DOCKER_LOGIN,
+        "docker pull ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}",
+    ]
+
+    for k in range(image_count):
+        commands.append("docker pull ${SCCI_CI_IMAGE_NAME_%s}:${SCCI_TAG}" % (k))
+
+    return commands
+
+
+def get_push_commands(image_count: int) -> CommandList:
+    commands: CommandList = [
+        DOCKER_LOGIN,
+    ]
+
+    for k in range(image_count):
+        commands.append(
+            "docker pull ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG}"
+            % (k)
+        )
+
+        commands.append(
+            "docker tag ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE_%s}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE_%s}:${SCCI_TAG}"
+            % (k, k)
+        )
+        commands.append(
+            "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE_%s}:${SCCI_TAG}"
+            % (k)
+        )
+
+    return commands
 
 
 def assemble_env_vars(
     repo_model: RepoModel,
     registries: Dict[str, RegistryEndpointyModel],
-    image_name: str,
     tag: str,
 ) -> Dict[str, str]:
     clone_directory: Path = Path(TemporaryDirectory().name)
 
     registry: RegistryEndpointyModel = registries[repo_model.registry.target]
-    test_image = repo_model.registry.local_to_test[image_name]
-    release_image = repo_model.registry.test_to_release[test_image]
 
-    return {
+    env_vars: Dict[str, str] = {
         "SCCI_REPO": repo_model.escaped_repo,
         "SCCI_CLONE_DIR": f"{clone_directory}",
-        "SCCI_IMAGE_NAME": image_name,
         "SCCI_TAG": tag,
-        "SCCI_TEST_IMAGE": test_image,
-        "SCCI_RELEASE_IMAGE": release_image,
         "SCCI_TARGET_REGISTRY_ADDRESS": registry.address,
         "SCCI_TARGET_REGISTRY_PASSWORD": registry.password.get_secret_value(),
         "SCCI_TARGET_REGISTRY_USER": registry.user,
     }
 
+    # for multiple image builds
+    local_images: List[str] = list(repo_model.registry.local_to_test.keys())
+    test_images: List[str] = list(repo_model.registry.local_to_test.values())
+    release_images: List[str] = list(repo_model.registry.test_to_release.values())
+
+    for k, image_name in enumerate(local_images):
+        env_vars[f"SCCI_IMAGE_NAME_{k}"] = image_name
+
+    for k, image_name in enumerate(test_images):
+        env_vars[f"SCCI_TEST_IMAGE_{k}"] = image_name
+
+    for k, image_name in enumerate(release_images):
+        env_vars[f"SCCI_RELEASE_IMAGE_{k}"] = image_name
+
+    return env_vars
+
 
 def validate_commands_list(
     commands_list: CommandList, env_vars: Dict[str, str]

src/docker_publisher_osparc_services/http_interface.py

@@ -39,7 +39,7 @@ async def github_did_last_repo_run_pass(
 async def _gitlab_get_project_id(repo_model: RepoModel) -> str:
     async with async_client() as client:
         parsed_url = URL(repo_model.address)
-        repo_name = parsed_url.path.split("/")[-1].strip(".git")
+        repo_name = parsed_url.path.split("/")[-1].replace(".git", "")
         host = parsed_url.host
         url = f"https://{host}/api/v4/projects?search={repo_name}"
         result = await client.get(
@@ -55,7 +55,7 @@ async def _gitlab_get_project_id(repo_model: RepoModel) -> str:
                 return repo["id"]
 
         message = (
-            f"Searching for {repo_name} did not yield the deisired result {found_repos}"
+            f"Searching for {repo_name} did not yield the deisired result {found_repos} {parsed_url}"
         )
         raise CouldNotFindAGitlabRepositoryRepoException(message)