From 08b964570f59b530424dbfb6aec29aa0184998f9 Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 17 Sep 2024 17:43:59 +0200 Subject: [PATCH] CH-100 refactoring, small fixes --- .dockerignore | 3 +- .gitignore | 2 +- .../compose/templates/auto-compose.yaml | 6 +- .../compose/templates/auto-gatekeepers.yaml | 6 +- .../ch_cli_tools/configurationgenerator.py | 23 ++++--- .../ch_cli_tools/dockercompose.py | 11 ++-- .../deployment-cli-tools/ch_cli_tools/helm.py | 2 +- .../ch_cli_tools/skaffold.py | 2 +- tools/deployment-cli-tools/harness-deployment | 64 +++++++------------ .../tests/test_dockercompose.py | 4 +- 10 files changed, 57 insertions(+), 66 deletions(-) diff --git a/.dockerignore b/.dockerignore index f98de6ccc..a5b6ffeda 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,4 +19,5 @@ __pycache__ /application-templates /deployment-configuration /cloud-harness -.openapi-generator \ No newline at end of file +.openapi-generator +docker-compose.yaml \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1c0fcc2e4..f4d38460f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ node_modules *.DS_Store deployment/helm deployment/compose -deployment/docker-compose.yaml +docker-compose.yaml *.egg-info *.idea /build diff --git a/deployment-configuration/compose/templates/auto-compose.yaml b/deployment-configuration/compose/templates/auto-compose.yaml index 46b076612..b00850854 100644 --- a/deployment-configuration/compose/templates/auto-compose.yaml +++ b/deployment-configuration/compose/templates/auto-compose.yaml @@ -163,9 +163,9 @@ services: {{- end }} {{- end }} volumes: - - ./compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro + - ./deployment/compose/allvalues.yaml:/opt/cloudharness/resources/allvalues.yaml:ro {{- range $file_name, $_ := $app_config.harness.secrets }} - - ./compose/resources/generated/auth/{{ $file_name }}:/opt/cloudharness/resources/auth/{{ $file_name }} + - ./deployment/compose/resources/generated/auth/{{ $file_name }}:/opt/cloudharness/resources/auth/{{ $file_name }} {{- end }} {{- if or $deployment.volume $app_config.harness.resources }} {{- with $deployment.volume }} @@ -176,7 +176,7 @@ services: {{- with $app_config.harness.resources }} {{- range .}} - type: bind - source: ./compose/resources/generated/{{ $app_name }}/{{ .src }} + source: ./deployment/compose/resources/generated/{{ $app_name }}/{{ .src }} target: {{ .dst }} {{- end }} {{- end}} diff --git a/deployment-configuration/compose/templates/auto-gatekeepers.yaml b/deployment-configuration/compose/templates/auto-gatekeepers.yaml index fcd7d19a7..4e6b7ac5c 100644 --- a/deployment-configuration/compose/templates/auto-gatekeepers.yaml +++ b/deployment-configuration/compose/templates/auto-gatekeepers.yaml @@ -23,9 +23,9 @@ environment: - PROXY_CONFIG_FILE=/opt/proxy.yml volumes: - - ./compose/resources/generated/{{ $gk_name }}/proxy.yml:/opt/proxy.yml - - ./compose/resources/generated/{{ $gk_name }}/cacert.crt:/etc/pki/ca-trust/source/anchors/cacert.crt - - ./compose/resources/generated/{{ $gk_name }}/access-denied.html.tmpl:/templates/access-denied.html.tmpl + - ./deployment/compose/resources/generated/{{ $gk_name }}/proxy.yml:/opt/proxy.yml + - ./deployment/compose/resources/generated/{{ $gk_name }}/cacert.crt:/etc/pki/ca-trust/source/anchors/cacert.crt + - ./deployment/compose/resources/generated/{{ $gk_name }}/access-denied.html.tmpl:/templates/access-denied.html.tmpl labels: - "traefik.enable=true" - "traefik.http.services.{{ $gk_name }}.loadbalancer.server.port={{ .app.harness.service.port }}" diff --git a/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py b/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py index 2228e3af8..56c4f9ac3 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py +++ b/tools/deployment-cli-tools/ch_cli_tools/configurationgenerator.py @@ -1,7 +1,7 @@ """ Utilities to create a helm chart from a CloudHarness directory structure """ -from typing import Union +from typing import List, Union import yaml import os import shutil @@ -10,7 +10,7 @@ import tarfile from docker import from_env as DockerClient from pathlib import Path - +import abc from . import HERE, CH_ROOT from cloudharness_utils.constants import TEST_IMAGES_PATH, HELM_CHART_PATH, APPS_PATH, HELM_PATH, \ @@ -32,10 +32,11 @@ DEFAULT_IGNORE = ('/tasks', '.dockerignore', '.hypothesis', "__pycache__", '.node_modules', 'dist', 'build', '.coverage') -class ConfigurationGenerator(object): - def __init__(self, root_paths, tag: Union[str, int, None] = 'latest', registry='', local=True, domain=None, exclude=(), secured=True, - output_path='./deployment', include=None, registry_secret=None, tls=True, env=None, - namespace=None, templates_path=HELM_PATH): +class ConfigurationGenerator(object, metaclass=abc.ABCMeta): + + def __init__(self, root_paths: List[str], tag: Union[str, int, None] = 'latest', registry='', local=True, domain=None, exclude=(), secured=True, + output_path='./deployment', include: List[str] = None, registry_secret: str = None, tls: str = True, env: str = None, + namespace: str = None, templates_path: str = HELM_PATH): assert domain, 'A domain must be specified' self.root_paths = [Path(r) for r in root_paths] self.tag = tag @@ -63,6 +64,10 @@ def __init__(self, root_paths, tag: Union[str, int, None] = 'latest', registry=' self.base_images = {} self.all_images = {} + @abc.abstractmethod + def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_values={}): + ... + def __init_deployment(self): """ Create the base helm chart @@ -94,11 +99,11 @@ def _process_applications(self, helm_values, base_image_name): app_base_path = root_path / APPS_PATH app_values = self.collect_app_values( - app_base_path, base_image_name=base_image_name) + app_base_path, base_image_name=base_image_name, helm_values=helm_values) helm_values[KEY_APPS] = dict_merge(helm_values[KEY_APPS], app_values) - def collect_app_values(self, app_base_path, base_image_name=None): + def collect_app_values(self, app_base_path, base_image_name=None, helm_values=None): values = {} for app_path in app_base_path.glob("*/"): # We get the sub-files that are directories @@ -108,7 +113,7 @@ def collect_app_values(self, app_base_path, base_image_name=None): continue app_key = app_name.replace('-', '_') - app_values = self.create_app_values_spec(app_name, app_path, base_image_name=base_image_name) + app_values = self.create_app_values_spec(app_name, app_path, base_image_name=base_image_name, helm_values=helm_values) # dockerfile_path = next(app_path.rglob('**/Dockerfile'), None) # # for dockerfile_path in app_path.rglob('**/Dockerfile'): diff --git a/tools/deployment-cli-tools/ch_cli_tools/dockercompose.py b/tools/deployment-cli-tools/ch_cli_tools/dockercompose.py index e714f1f1b..87ecf5b4d 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/dockercompose.py +++ b/tools/deployment-cli-tools/ch_cli_tools/dockercompose.py @@ -84,7 +84,7 @@ def process_values(self) -> HarnessMainConfig: def generate_docker_compose_yaml(self): compose_templates = self.dest_deployment_path - dest_compose_yaml = self.dest_deployment_path.parent / "docker-compose.yaml" + dest_compose_yaml = self.dest_deployment_path.parent.parent / "docker-compose.yaml" logging.info(f'Generate docker compose configuration in: {dest_compose_yaml}, using templates from {compose_templates}') command = f"helm template {compose_templates} > {dest_compose_yaml}" @@ -204,7 +204,7 @@ def __finish_helm_values(self, values): create_env_variables(values) return values, self.include - def create_app_values_spec(self, app_name, app_path, base_image_name=None): + def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_values={}): logging.info('Generating values script for ' + app_name) deploy_path = app_path / 'deploy' @@ -231,9 +231,10 @@ def create_app_values_spec(self, app_name, app_path, base_image_name=None): image_paths = [path for path in find_dockerfiles_paths( app_path) if 'tasks/' not in path and 'subapps' not in path] - # Inject entry points commands - for image_path in image_paths: - self.inject_entry_points_commands(values, image_path, app_path) + # Inject entry points commands to enable debug + if helm_values.get("debug", False): + for image_path in image_paths: + self.inject_entry_points_commands(values, image_path, app_path) if len(image_paths) > 1: logging.warning('Multiple Dockerfiles found in application %s. Picking the first one: %s', app_name, diff --git a/tools/deployment-cli-tools/ch_cli_tools/helm.py b/tools/deployment-cli-tools/ch_cli_tools/helm.py index d1725c67e..b81d0f191 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/helm.py +++ b/tools/deployment-cli-tools/ch_cli_tools/helm.py @@ -155,7 +155,7 @@ def __finish_helm_values(self, values): create_env_variables(values) return values, self.include - def create_app_values_spec(self, app_name, app_path, base_image_name=None): + def create_app_values_spec(self, app_name, app_path, base_image_name=None, helm_values={}): logging.info('Generating values script for ' + app_name) specific_template_path = os.path.join(app_path, 'deploy', 'values.yaml') diff --git a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py index 0c471f422..22320082a 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py +++ b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py @@ -231,7 +231,7 @@ def identify_unicorn_based_main(candidates): skaffold_conf['build']['artifacts'] = [v for v in artifacts.values()] - with open('skaffold.yaml', "w") as f: + with open(os.path.join(output_path, 'skaffold.yaml'), "w") as f: yaml.dump(skaffold_conf, f) return skaffold_conf diff --git a/tools/deployment-cli-tools/harness-deployment b/tools/deployment-cli-tools/harness-deployment index 908caed0e..f4c9fe3db 100644 --- a/tools/deployment-cli-tools/harness-deployment +++ b/tools/deployment-cli-tools/harness-deployment @@ -11,7 +11,7 @@ from ch_cli_tools.skaffold import create_skaffold_configuration, create_vscode_d from ch_cli_tools.codefresh import create_codefresh_deployment_scripts, write_env_file from ch_cli_tools.preprocessing import preprocess_build_overrides from ch_cli_tools.utils import merge_app_directories -from cloudharness_utils.constants import DEPLOYMENT_PATH, COMPOSE_ENGINE +from cloudharness_utils.constants import DEPLOYMENT_PATH, COMPOSE_ENGINE, HELM_ENGINE HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') ROOT = os.path.dirname(os.path.dirname(HERE)).replace(os.path.sep, '/') @@ -79,48 +79,33 @@ if __name__ == "__main__": else: if args.merge: - logging.warn( + logging.warning( "Merge (-m, --merge) argument is deprecated. Directory merging is now set automatically") merge_app_directories(root_paths, destination=args.merge) root_paths = [args.merge] - if not args.docker_compose: - helm_values = create_helm_chart( - root_paths, - tag=args.tag, - registry=args.registry, - domain=args.domain, - local=args.local, - secured=not args.unsecured, - output_path=args.output_path, - exclude=args.exclude, - include=args.include, - registry_secret=args.registry_secret, - tls=not args.no_tls, - env=envs, - namespace=args.namespace - ) - else: - helm_values = create_docker_compose_configuration( - root_paths, - tag=args.tag, - registry=args.registry, - domain=args.domain, - local=args.local, - secured=not args.unsecured, - output_path=args.output_path, - exclude=args.exclude, - include=args.include, - registry_secret=args.registry_secret, - tls=not args.no_tls, - env=envs, - namespace=args.namespace, - ) + chart_fn = create_helm_chart if not args.docker_compose else create_docker_compose_configuration + + helm_values = chart_fn( + root_paths, + tag=args.tag, + registry=args.registry, + domain=args.domain, + local=args.local, + secured=not args.unsecured, + output_path=args.output_path, + exclude=args.exclude, + include=args.include, + registry_secret=args.registry_secret, + tls=not args.no_tls, + env=envs, + namespace=args.namespace + ) merged_root_paths = preprocess_build_overrides( root_paths=root_paths, helm_values=helm_values) - if not args.no_cd_gen and envs: + if not args.no_cd_gen and envs and not args.docker_compose: create_codefresh_deployment_scripts( merged_root_paths, include=args.include, @@ -132,13 +117,12 @@ if __name__ == "__main__": if args.write_env: write_env_file(helm_values, os.path.join(root_paths[-1], DEPLOYMENT_PATH, ".env")) + create_skaffold_configuration(merged_root_paths, helm_values, backend_deploy=COMPOSE_ENGINE if args.docker_compose else HELM_ENGINE) + if not args.docker_compose: - create_skaffold_configuration(merged_root_paths, helm_values) - else: - create_skaffold_configuration(merged_root_paths, helm_values, backend_deploy=COMPOSE_ENGINE) - create_vscode_debug_configuration(root_paths, helm_values) + create_vscode_debug_configuration(root_paths, helm_values) - hosts_info(helm_values) + hosts_info(helm_values) if args.deploy: deploy(args.namespace, args.output_path) diff --git a/tools/deployment-cli-tools/tests/test_dockercompose.py b/tools/deployment-cli-tools/tests/test_dockercompose.py index 3e2c23005..13959434c 100644 --- a/tools/deployment-cli-tools/tests/test_dockercompose.py +++ b/tools/deployment-cli-tools/tests/test_dockercompose.py @@ -67,7 +67,7 @@ def test_collect_compose_values(tmp_path): compose_path = out_folder / COMPOSE_PATH # Check files - assert exists(out_folder / 'docker-compose.yaml') + assert exists(out_folder.parent / 'docker-compose.yaml') assert exists(compose_path) assert exists(compose_path / 'values.yaml') assert exists(compose_path / 'allvalues.yaml') @@ -136,7 +136,7 @@ def test_collect_compose_values_noreg_noinclude(tmp_path): compose_path = out_path / COMPOSE_PATH # Check files - assert exists(out_path / 'docker-compose.yaml') + assert exists(out_path.parent / 'docker-compose.yaml') assert exists(compose_path) assert exists(compose_path / 'values.yaml') assert exists(compose_path / 'allvalues.yaml')