refactor CI actions to push to correct targets

src/docker_publisher_osparc_services/cli.py

@@ -4,23 +4,14 @@
 import click
 
 from . import __version__
-from .gitlab_ci_setup.commands import (
-    COMMANDS_BUILD,
-    COMMANDS_PUSH,
-    COMMANDS_TEST_BASE,
-    assemble_env_vars,
-    validate_commands_list,
-)
+from .gitlab_ci_setup.commands import (COMMANDS_BUILD, COMMANDS_PUSH,
+                                       COMMANDS_TEST_BASE, assemble_env_vars,
+                                       validate_commands_list)
 from .gitlab_ci_setup.pipeline_config import PipelineConfig, PipelineGenerator
 from .http_interface import get_tags_for_repo
 from .models import ConfigModel
-from .operations import (
-    assemble_compose,
-    clone_repo,
-    did_ci_pass,
-    fetch_images_from_compose_spec,
-    get_branch_hash,
-)
+from .operations import (assemble_compose, clone_repo, did_ci_pass,
+                         fetch_images_from_compose_spec, get_branch_hash)
 
 
 async def run_command(config: Path) -> None:
@@ -75,7 +66,10 @@ async def run_command(config: Path) -> None:
 
                     # build commands validation
                     env_vars = assemble_env_vars(
-                        repo_model, image_name, remote_name, tag
+                        repo_model=repo_model,
+                        registries=cfg.registries,
+                        image_name=image_name,
+                        tag=tag,
                     )
                     validate_commands_list(COMMANDS_BUILD, env_vars)
 

src/docker_publisher_osparc_services/gitlab_ci_setup/commands.py

@@ -3,9 +3,12 @@
 from tempfile import TemporaryDirectory
 from typing import Dict, List
 
-from ..models import RepoModel
+from ..models import RegistryEndpointyModel, RepoModel
 
-PREFIX = "SCCI"  # short for simcore ci
+DOCKER_LOGIN: str = (
+    "echo '${SCCI_TARGET_REGISTRY_PASSWORD}' | "
+    "docker login ${SCCI_TARGET_REGISTRY_ADDRESS} --username ${SCCI_TARGET_REGISTRY_USER} --password-stdin"
+)
 
 CommandList = List[str]
 
@@ -14,35 +17,49 @@
     "cd ${SCCI_CLONE_DIR}",
     "ooil compose",
     "docker-compose build",
-    "docker tag ${SCCI_IMAGE_NAME}:${SCCI_TAG} ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}",
-    "docker push ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}",
+    DOCKER_LOGIN,
+    "docker tag ${SCCI_IMAGE_NAME}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}",
+    "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}",
 ]
 
 COMMANDS_TEST_BASE: CommandList = [
     "git clone ${SCCI_REPO} ${SCCI_CLONE_DIR}",
     "cd ${SCCI_CLONE_DIR}",
+    DOCKER_LOGIN,
     "docker pull ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}",
     # if user defines extra commands those will be append here
 ]
 
 COMMANDS_PUSH: CommandList = [
-    "docker pull ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG}",
-    "docker tag ${SCCI_CI_IMAGE_NAME}:${SCCI_TAG} ${SCCI_REMOTE_NAME}:${SCCI_TAG}",
-    "docker push ${SCCI_REMOTE_NAME}:${SCCI_TAG}",
+    DOCKER_LOGIN,
+    "docker pull ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG}",
+    "docker tag ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_TEST_IMAGE}:${SCCI_TAG} ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE}:${SCCI_TAG}",
+    "docker push ${SCCI_TARGET_REGISTRY_ADDRESS}/${SCCI_RELEASE_IMAGE}:${SCCI_TAG}",
 ]
 
 
 def assemble_env_vars(
-    repo_model: RepoModel, image_name: str, remote_name: str, tag: str
+    repo_model: RepoModel,
+    registries: Dict[str, RegistryEndpointyModel],
+    image_name: str,
+    tag: str,
 ) -> Dict[str, str]:
     clone_directory: Path = Path(TemporaryDirectory().name)
+
+    registry: RegistryEndpointyModel = registries[repo_model.registry.target]
+    test_image = repo_model.registry.local_to_test[image_name]
+    release_image = repo_model.registry.test_to_release[test_image]
+
     return {
-        f"{PREFIX}_REPO": repo_model.repo,
-        f"{PREFIX}_CLONE_DIR": f"{clone_directory}",
-        f"{PREFIX}_IMAGE_NAME": image_name,
-        f"{PREFIX}_TAG": tag,
-        f"{PREFIX}_CI_IMAGE_NAME": f"ci-test/{image_name}",
-        f"{PREFIX}_REMOTE_NAME": remote_name,
+        "SCCI_REPO": repo_model.repo,
+        "SCCI_CLONE_DIR": f"{clone_directory}",
+        "SCCI_IMAGE_NAME": image_name,
+        "SCCI_TAG": tag,
+        "SCCI_TEST_IMAGE": test_image,
+        "SCCI_RELEASE_IMAGE": release_image,
+        "SCCI_TARGET_REGISTRY_ADDRESS": registry.address,
+        "SCCI_TARGET_REGISTRY_PASSWORD": registry.password,
+        "SCCI_TARGET_REGISTRY_USER": registry.user,
     }
 
 
@@ -53,7 +70,7 @@ def validate_commands_list(
     for command in commands_list:
         hits = re.findall(r"\$\{(.*?)\}", command)
         for hit in hits:
-            if hit.startswith(PREFIX) and hit not in env_vars:
+            if hit.startswith("SCCI") and hit not in env_vars:
                 raise ValueError(
                     f"env var '{hit}'\ndefined in '{command}'\n "
                     f"not found default injected env vars '{env_vars}'"

src/docker_publisher_osparc_services/models.py

@@ -24,13 +24,37 @@ class ImageSrcDstModel(BaseModel):
 
 class RegistryTargetModel(BaseModel):
     target: str
-    local_to_remote: Dict[str, str] = Field(
+    local_to_test: Dict[str, str] = Field(
         ...,
-        description="mapping between locally built image name and remote name in registry",
+        description="mapping between: `local build` image and `remote test` image",
         example={
-            "simcore/services/dynamic/jupyter-math": "ci/osparc-sparc-internal/master/jupyter-math"
+            "simcore/services/dynamic/jupyter-math": "ci/builder/osparc-sparc-internal/master/jupyter-math"
         },
     )
+    test_to_release: Dict[str, str] = Field(
+        ...,
+        description="mapping between: `remote test` image test image and `remote release` image",
+        example={
+            "ci/builder/osparc-sparc-internal/master/jupyter-math": "ci/osparc-sparc-internal/master/jupyter-math"
+        },
+    )
+
+    @root_validator()
+    def validate_consistency(cls, values: Dict) -> Dict:
+        local_to_test = values["local_to_test"]
+        test_to_release = values["test_to_release"]
+        if len(local_to_test) != len(test_to_release):
+            raise ValueError(
+                f"Following dicts should have the same entry count {local_to_test=}, {test_to_release=}"
+            )
+
+        for test_image in local_to_test.values():
+            if test_image not in test_to_release:
+                raise ValueError(
+                    f"Expected {test_image=} to be found in {test_to_release=}"
+                )
+
+        return values
 
 
 class GitLabModel(BaseModel):

src/docker_publisher_osparc_services/operations.py

@@ -5,7 +5,8 @@
 import yaml
 
 from .exceptions import BaseAppException, GITCommitHashInvalid
-from .http_interface import github_did_last_repo_run_pass, gitlab_did_last_repo_run_pass
+from .http_interface import (github_did_last_repo_run_pass,
+                             gitlab_did_last_repo_run_pass)
 from .models import HostType, RepoModel
 from .utils import command_output