CH-100 refactoring, small fixes

.dockerignore

@@ -19,4 +19,5 @@ __pycache__
 /application-templates
 /deployment-configuration
 /cloud-harness
-.openapi-generator
+.openapi-generator
+docker-compose.yaml

.gitignore

@@ -8,7 +8,7 @@ node_modules
 *.DS_Store
 deployment/helm
 deployment/compose
-deployment/docker-compose.yaml
+docker-compose.yaml
 *.egg-info
 *.idea
 /build

deployment-configuration/compose/templates/auto-compose.yaml

@@ -163,9 +163,9 @@ services:
     {{- end }}
     {{- end }}
     volumes:
-      - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro
+      - ./deployment/compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro
     {{- range $file_name, $_ := $app_config.harness.secrets }}
-      - ./compose/resources/generated/auth/{{ $file_name }}:/opt/cloudharness/resources/auth/{{ $file_name }}
+      - ./deployment/compose/resources/generated/auth/{{ $file_name }}:/opt/cloudharness/resources/auth/{{ $file_name }}
     {{- end }}
     {{- if or $deployment.volume $app_config.harness.resources }}
     {{- with $deployment.volume }}
@@ -176,7 +176,7 @@ services:
     {{- with $app_config.harness.resources }}
     {{- range .}}
       - type: bind
-        source: ./compose/resources/generated/{{ $app_name }}/{{ .src }}
+        source: ./deployment/compose/resources/generated/{{ $app_name }}/{{ .src }}
         target: {{ .dst }}
     {{- end }}
     {{- end}}

deployment-configuration/compose/templates/auto-gatekeepers.yaml

@@ -23,9 +23,9 @@
   environment:
     - PROXY_CONFIG_FILE=/opt/proxy.yml
   volumes:
-    - ./compose/resources/generated/{{ $gk_name }}/proxy.yml:/opt/proxy.yml
-    - ./compose/resources/generated/{{ $gk_name }}/cacert.crt:/etc/pki/ca-trust/source/anchors/cacert.crt
-    - ./compose/resources/generated/{{ $gk_name }}/access-denied.html.tmpl:/templates/access-denied.html.tmpl
+    - ./deployment/compose/resources/generated/{{ $gk_name }}/proxy.yml:/opt/proxy.yml
+    - ./deployment/compose/resources/generated/{{ $gk_name }}/cacert.crt:/etc/pki/ca-trust/source/anchors/cacert.crt
+    - ./deployment/compose/resources/generated/{{ $gk_name }}/access-denied.html.tmpl:/templates/access-denied.html.tmpl
   labels:
     - "traefik.enable=true"
     - "traefik.http.services.{{ $gk_name }}.loadbalancer.server.port={{ .app.harness.service.port }}"

tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py

@@ -1,7 +1,7 @@
 """
 Utilities to create a helm chart from a CloudHarness directory structure
 """
-from typing import Union
+from typing import List, Union
 import yaml
 import os
 import shutil
@@ -10,7 +10,7 @@
 import tarfile
 from docker import from_env as DockerClient
 from pathlib import Path
-
+import abc
 
 from . import HERE, CH_ROOT
 from cloudharness_utils.constants import TEST_IMAGES_PATH, HELM_CHART_PATH, APPS_PATH, HELM_PATH, \
@@ -32,10 +32,11 @@
 DEFAULT_IGNORE = ('/tasks', '.dockerignore', '.hypothesis', "__pycache__", '.node_modules', 'dist', 'build', '.coverage')
 
 
-class ConfigurationGenerator(object):
-    def __init__(self, root_paths, tag: Union[str, int, None] = 'latest', registry='', local=True, domain=None, exclude=(), secured=True,
-                 output_path='./deployment', include=None, registry_secret=None, tls=True, env=None,
-                 namespace=None, templates_path=HELM_PATH):
+class ConfigurationGenerator(object, metaclass=abc.ABCMeta):
+
+    def __init__(self, root_paths: List[str], tag: Union[str, int, None] = 'latest', registry='', local=True, domain=None, exclude=(), secured=True,
+                 output_path='./deployment', include: List[str] = None, registry_secret: str = None, tls: str = True, env: str = None,
+                 namespace: str = None, templates_path: str = HELM_PATH):
         assert domain, 'A domain must be specified'
         self.root_paths = [Path(r) for r in root_paths]
         self.tag = tag
@@ -63,6 +64,10 @@ def __init__(self, root_paths, tag: Union[str, int, None] = 'latest', registry='
         self.base_images = {}
         self.all_images = {}
 
+    @abc.abstractmethod
+    def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_values={}):
+        ...
+
     def __init_deployment(self):
         """
         Create the base helm chart
@@ -94,11 +99,11 @@ def _process_applications(self, helm_values, base_image_name):
 
             app_base_path = root_path / APPS_PATH
             app_values = self.collect_app_values(
-                app_base_path, base_image_name=base_image_name)
+                app_base_path, base_image_name=base_image_name, helm_values=helm_values)
             helm_values[KEY_APPS] = dict_merge(helm_values[KEY_APPS],
                                                app_values)
 
-    def collect_app_values(self, app_base_path, base_image_name=None):
+    def collect_app_values(self, app_base_path, base_image_name=None, helm_values=None):
         values = {}
 
         for app_path in app_base_path.glob("*/"):  # We get the sub-files that are directories
@@ -108,7 +113,7 @@ def collect_app_values(self, app_base_path, base_image_name=None):
                 continue
             app_key = app_name.replace('-', '_')
 
-            app_values = self.create_app_values_spec(app_name, app_path, base_image_name=base_image_name)
+            app_values = self.create_app_values_spec(app_name, app_path, base_image_name=base_image_name, helm_values=helm_values)
 
             # dockerfile_path = next(app_path.rglob('**/Dockerfile'), None)
             # # for dockerfile_path in app_path.rglob('**/Dockerfile'):

tools/deployment-cli-tools/ch_cli_tools/dockercompose.py

@@ -84,7 +84,7 @@ def process_values(self) -> HarnessMainConfig:
 
     def generate_docker_compose_yaml(self):
         compose_templates = self.dest_deployment_path
-        dest_compose_yaml = self.dest_deployment_path.parent / "docker-compose.yaml"
+        dest_compose_yaml = self.dest_deployment_path.parent.parent / "docker-compose.yaml"
 
         logging.info(f'Generate docker compose configuration in: {dest_compose_yaml}, using templates from {compose_templates}')
         command = f"helm template {compose_templates} > {dest_compose_yaml}"
@@ -204,7 +204,7 @@ def __finish_helm_values(self, values):
         create_env_variables(values)
         return values, self.include
 
-    def create_app_values_spec(self, app_name, app_path, base_image_name=None):
+    def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_values={}):
         logging.info('Generating values script for ' + app_name)
 
         deploy_path = app_path / 'deploy'
@@ -231,9 +231,10 @@ def create_app_values_spec(self, app_name, app_path, base_image_name=None):
         image_paths = [path for path in find_dockerfiles_paths(
             app_path) if 'tasks/' not in path and 'subapps' not in path]
 
-        # Inject entry points commands
-        for image_path in image_paths:
-            self.inject_entry_points_commands(values, image_path, app_path)
+        # Inject entry points commands to enable debug
+        if helm_values.get("debug", False):
+            for image_path in image_paths:
+                self.inject_entry_points_commands(values, image_path, app_path)
 
         if len(image_paths) > 1:
             logging.warning('Multiple Dockerfiles found in application %s. Picking the first one: %s', app_name,

tools/deployment-cli-tools/ch_cli_tools/helm.py

@@ -155,7 +155,7 @@ def __finish_helm_values(self, values):
         create_env_variables(values)
         return values, self.include
 
-    def create_app_values_spec(self, app_name, app_path, base_image_name=None):
+    def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_values={}):
         logging.info('Generating values script for ' + app_name)
 
         specific_template_path = os.path.join(app_path, 'deploy', 'values.yaml')

tools/deployment-cli-tools/ch_cli_tools/skaffold.py

@@ -231,7 +231,7 @@ def identify_unicorn_based_main(candidates):
 
     skaffold_conf['build']['artifacts'] = [v for v in artifacts.values()]
 
-    with open('skaffold.yaml', "w") as f:
+    with open(os.path.join(output_path, 'skaffold.yaml'), "w") as f:
         yaml.dump(skaffold_conf, f)
     return skaffold_conf
 

tools/deployment-cli-tools/harness-deployment

@@ -11,7 +11,7 @@ from ch_cli_tools.skaffold import create_skaffold_configuration, create_vscode_d
 from ch_cli_tools.codefresh import create_codefresh_deployment_scripts, write_env_file
 from ch_cli_tools.preprocessing import preprocess_build_overrides
 from ch_cli_tools.utils import merge_app_directories
-from cloudharness_utils.constants import DEPLOYMENT_PATH, COMPOSE_ENGINE
+from cloudharness_utils.constants import DEPLOYMENT_PATH, COMPOSE_ENGINE, HELM_ENGINE
 
 HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/')
 ROOT = os.path.dirname(os.path.dirname(HERE)).replace(os.path.sep, '/')
@@ -79,48 +79,33 @@ if __name__ == "__main__":
     else:
 
         if args.merge:
-            logging.warn(
+            logging.warning(
                 "Merge (-m, --merge) argument is deprecated. Directory merging is now set automatically")
             merge_app_directories(root_paths, destination=args.merge)
             root_paths = [args.merge]
 
-        if not args.docker_compose:
-            helm_values = create_helm_chart(
-                root_paths,
-                tag=args.tag,
-                registry=args.registry,
-                domain=args.domain,
-                local=args.local,
-                secured=not args.unsecured,
-                output_path=args.output_path,
-                exclude=args.exclude,
-                include=args.include,
-                registry_secret=args.registry_secret,
-                tls=not args.no_tls,
-                env=envs,
-                namespace=args.namespace
-            )
-        else:
-            helm_values = create_docker_compose_configuration(
-                root_paths,
-                tag=args.tag,
-                registry=args.registry,
-                domain=args.domain,
-                local=args.local,
-                secured=not args.unsecured,
-                output_path=args.output_path,
-                exclude=args.exclude,
-                include=args.include,
-                registry_secret=args.registry_secret,
-                tls=not args.no_tls,
-                env=envs,
-                namespace=args.namespace,
-            )
+        chart_fn = create_helm_chart if not args.docker_compose else create_docker_compose_configuration
+
+        helm_values = chart_fn(
+            root_paths,
+            tag=args.tag,
+            registry=args.registry,
+            domain=args.domain,
+            local=args.local,
+            secured=not args.unsecured,
+            output_path=args.output_path,
+            exclude=args.exclude,
+            include=args.include,
+            registry_secret=args.registry_secret,
+            tls=not args.no_tls,
+            env=envs,
+            namespace=args.namespace
+        )
 
         merged_root_paths = preprocess_build_overrides(
             root_paths=root_paths, helm_values=helm_values)
 
-        if not args.no_cd_gen and envs:
+        if not args.no_cd_gen and envs and not args.docker_compose:
             create_codefresh_deployment_scripts(
                 merged_root_paths,
                 include=args.include,
@@ -132,13 +117,12 @@ if __name__ == "__main__":
         if args.write_env:
             write_env_file(helm_values, os.path.join(root_paths[-1], DEPLOYMENT_PATH, ".env"))
 
+        create_skaffold_configuration(merged_root_paths, helm_values, backend_deploy=COMPOSE_ENGINE if args.docker_compose else HELM_ENGINE)
+
         if not args.docker_compose:
-            create_skaffold_configuration(merged_root_paths, helm_values)
-        else:
-            create_skaffold_configuration(merged_root_paths, helm_values, backend_deploy=COMPOSE_ENGINE)
-        create_vscode_debug_configuration(root_paths, helm_values)
+            create_vscode_debug_configuration(root_paths, helm_values)
 
-        hosts_info(helm_values)
+            hosts_info(helm_values)
 
     if args.deploy:
         deploy(args.namespace, args.output_path)

tools/deployment-cli-tools/tests/test_dockercompose.py

@@ -67,7 +67,7 @@ def test_collect_compose_values(tmp_path):
     compose_path = out_folder / COMPOSE_PATH
 
     # Check files
-    assert exists(out_folder / 'docker-compose.yaml')
+    assert exists(out_folder.parent / 'docker-compose.yaml')
     assert exists(compose_path)
     assert exists(compose_path / 'values.yaml')
     assert exists(compose_path / 'allvalues.yaml')
@@ -136,7 +136,7 @@ def test_collect_compose_values_noreg_noinclude(tmp_path):
     compose_path = out_path / COMPOSE_PATH
 
     # Check files
-    assert exists(out_path / 'docker-compose.yaml')
+    assert exists(out_path.parent / 'docker-compose.yaml')
     assert exists(compose_path)
     assert exists(compose_path / 'values.yaml')
     assert exists(compose_path / 'allvalues.yaml')