diff --git a/Pipfile.lock b/Pipfile.lock index 519481c..8223728 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -18,10 +18,10 @@ "default": { "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "chardet": { "hashes": [ @@ -186,13 +186,6 @@ } }, "develop": { - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, "attrs": { "hashes": [ "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", @@ -222,10 +215,10 @@ }, "certifi": { "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" ], - "version": "==2019.9.11" + "version": "==2019.11.28" }, "cgroupspy": { "hashes": [ @@ -365,11 +358,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", - "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + "sha256:3a8b2dfd0a2c6a3636e7c016a7e54ae04b997d30e69d5eacdca7a6c2221a1402", + "sha256:41e688146d000891f32b1669e8573c57e39e5060e7f5f647aa617cd9a9568278" ], "markers": "python_version < '3.8'", - "version": "==0.23" + "version": "==1.2.0" }, "itsdangerous": { "hashes": [ @@ -449,10 +442,10 @@ }, "more-itertools": { "hashes": [ - "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", - "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", + "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" ], - "version": "==7.2.0" + "version": "==8.0.0" }, "mypy-extensions": { "hashes": [ @@ -470,10 +463,10 @@ }, "pluggy": { "hashes": [ - "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", - "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.13.0" + "version": "==0.13.1" }, "protobuf": { "hashes": [ @@ -500,19 +493,19 @@ }, "psutil": { "hashes": [ - "sha256:021d361439586a0fd8e64f8392eb7da27135db980f249329f1a347b9de99c695", - "sha256:145e0f3ab9138165f9e156c307100905fd5d9b7227504b8a9d3417351052dc3d", - "sha256:348ad4179938c965a27d29cbda4a81a1b2c778ecd330a221aadc7bd33681afbd", - "sha256:3feea46fbd634a93437b718518d15b5dd49599dfb59a30c739e201cc79bb759d", - "sha256:474e10a92eeb4100c276d4cc67687adeb9d280bbca01031a3e41fb35dfc1d131", - "sha256:47aeb4280e80f27878caae4b572b29f0ec7967554b701ba33cd3720b17ba1b07", - "sha256:73a7e002781bc42fd014dfebb3fc0e45f8d92a4fb9da18baea6fb279fbc1d966", - "sha256:d051532ac944f1be0179e0506f6889833cf96e466262523e57a871de65a15147", - "sha256:dfb8c5c78579c226841908b539c2374da54da648ee5a837a731aa6a105a54c00", - "sha256:e3f5f9278867e95970854e92d0f5fe53af742a7fc4f2eba986943345bcaed05d", - "sha256:e9649bb8fc5cea1f7723af53e4212056a6f984ee31784c10632607f472dec5ee" + "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b", + "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806", + "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b", + "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995", + "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd", + "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73", + "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465", + "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d", + "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a", + "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217", + "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa" ], - "version": "==5.6.5" + "version": "==5.6.7" }, "py": { "hashes": [ @@ -536,10 +529,10 @@ }, "pytest": { "hashes": [ - "sha256:8e256fe71eb74e14a4d20a5987bb5e1488f0511ee800680aaedc62b9358714e8", - "sha256:ff0090819f669aaa0284d0f4aad1a6d9d67a6efdc6dd4eb4ac56b704f890a0d6" + "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418", + "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427" ], - "version": "==5.2.4" + "version": "==5.3.1" }, "pytest-mock": { "hashes": [ diff --git a/VERSION b/VERSION index 9c1218c..1b87bcd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.3 \ No newline at end of file +1.1.4 \ No newline at end of file diff --git a/cli/kaos_cli/commands/build.py b/cli/kaos_cli/commands/build.py index 57dfe1f..94b919a 100644 --- a/cli/kaos_cli/commands/build.py +++ b/cli/kaos_cli/commands/build.py @@ -7,6 +7,7 @@ from kaos_cli.facades.backend_facade import BackendFacade, is_cloud_provider from typing import Optional +from kaos_cli.utils.helpers import build_dir from kaos_cli.utils.custom_classes import CustomHelpOrder, NotRequiredIf from kaos_cli.utils.decorators import build_env_check, pass_obj from kaos_cli.utils.validators import validate_unused_port, validate_inputs, EnvironmentState @@ -16,8 +17,13 @@ # BUILD group # ============= @click.group(name='build', cls=CustomHelpOrder, - short_help=' {} and its {} '.format( - click.style('build', bold=True), click.style('sub-commands', bold=True))) + short_help='{} and its {}: {}, {}, {} and {}'.format( + click.style('Infrastructure deployments', bold=True), + click.style('sub-commands', bold=False), + click.style('deploy', bold=True), + click.style('list', bold=True), + click.style('set', bold=True), + click.style('active', bold=True))) def build(): """ Build command allows you to deploy infrastructre and list the available deployments @@ -26,8 +32,9 @@ def build(): @build.command(name='deploy', - short_help='{}'.format( - click.style('Build the kaos backend', bold=True, fg='black'))) + short_help='{} the {} backend'.format( + click.style('Build', bold=True), + click.style('kaos', bold=True))) @click.option('-c', '--cloud', type=click.Choice([DOCKER, MINIKUBE, AWS, GCP]), help='selected provider', required=True) @click.option('-e', '--env', type=click.Choice(['prod', 'stage', 'dev']), @@ -58,9 +65,12 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b click.echo('{} - Performing {} build of the backend'.format( click.style("Warning", bold=True, fg='yellow'), click.style("force", bold=True))) + env_state.remove_terraform_files() # set env variable appropriately env_state.set_build_env() + cloud = env_state.cloud + env = env_state.env if not yes: # confirm creation of backend @@ -106,7 +116,6 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b sys.exit(1) try: - is_built_successfully, env_state = backend.build(env_state.cloud, env_state.env, local_backend=local_backend, @@ -129,7 +138,7 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b click.echo("{} - Successfully built {} [{}] environment".format( click.style("Info", bold=True, fg='green'), click.style('kaos', bold=True), - click.style(env_state.env, bold=True, fg='blue'))) + click.style(env, bold=True, fg='blue'))) else: click.echo("{} - Successfully built {} environment".format( click.style("Info", bold=True, fg='green'), @@ -139,8 +148,8 @@ def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: b click.echo("{} - Deployment Unsuccessful while creating {} [{} {}] environment".format( click.style("Error", bold=True, fg='red'), click.style('kaos', bold=True), - click.style(env_state.cloud, bold=True, fg='red'), - click.style(env_state.env, bold=True, fg='red'))), + click.style(cloud, bold=True, fg='red'), + click.style(env, bold=True, fg='red'))), sys.exit(1) except Exception as e: @@ -250,8 +259,9 @@ def get_active_context(backend: BackendFacade): @click.command(name='destroy', - short_help='{}'.format( - click.style('Destroy the kaos backend', bold=True, fg='black'))) + short_help='{} the {} backend'.format( + click.style('Destroys', bold=True), + click.style('kaos', bold=True))) @click.option('-c', '--cloud', type=click.Choice([DOCKER, MINIKUBE, AWS, GCP]), help='selected provider provider', required=True) @click.option('-e', '--env', type=click.Choice(['prod', 'stage', 'dev']), @@ -272,7 +282,7 @@ def destroy(backend: BackendFacade, cloud, env, verbose, yes): env_state.set_build_env() # Ensure that appropriate warnings are displayed - env_state.validate_if_tfstate_exits() + env_state.validate_if_tf_state_exits() if not yes: # confirm creation of backend diff --git a/cli/kaos_cli/facades/backend_facade.py b/cli/kaos_cli/facades/backend_facade.py index a7fcbfa..cf63526 100644 --- a/cli/kaos_cli/facades/backend_facade.py +++ b/cli/kaos_cli/facades/backend_facade.py @@ -5,19 +5,14 @@ from distutils.dir_util import copy_tree import requests -from kaos_cli.constants import DOCKER, MINIKUBE, PROVIDER_DICT, AWS, BACKEND, INFRASTRUCTURE, GCP, LOCAL_CONFIG_DICT, \ +from kaos_cli.constants import DOCKER, MINIKUBE, AWS, BACKEND, INFRASTRUCTURE, GCP, LOCAL_CONFIG_DICT, \ CONTEXTS, ACTIVE, BACKEND_CACHE, DEFAULT, USER, REMOTE, KAOS_STATE_DIR from kaos_cli.exceptions.exceptions import HostnameError from kaos_cli.services.state_service import StateService from kaos_cli.services.terraform_service import TerraformService from kaos_cli.utils.environment import check_environment from kaos_cli.utils.helpers import build_dir -from kaos_cli.utils.validators import EnvironmentState -from kaos_cli.exceptions.handle_exceptions import handle_specific_exception, handle_exception - - -def is_cloud_provider(cloud): - return cloud not in (DOCKER, MINIKUBE) +from kaos_cli.utils.validators import EnvironmentState, is_cloud_provider class BackendFacade: @@ -134,17 +129,18 @@ def set_context_by_index(self, index): def build(self, provider, env, local_backend=False, verbose=False): env_state = EnvironmentState.initialize(provider, env) - if not env_state.if_build_dir_exists: + + if not os.path.exists(env_state.build_dir): build_dir(env_state.build_dir) auth_token = uuid.uuid4() extra_vars = self._get_vars(provider, env_state.build_dir, auth_token) - self.tf_service.cd_dir(env_state.build_dir) + self.tf_service.cd_dir(env_state.build_dir) self.tf_service.set_verbose(verbose) - directory = self._tf_init(provider, env, local_backend, destroying=False) - self.tf_service.plan(directory, extra_vars) - self.tf_service.apply(directory, extra_vars) + self._tf_init(env_state, provider, env, local_backend, destroying=False) + self.tf_service.plan(env_state.build_dir, extra_vars) + self.tf_service.apply(env_state.build_dir, extra_vars) self.tf_service.execute() # check if the deployed successfully @@ -153,20 +149,15 @@ def build(self, provider, env, local_backend=False, verbose=False): if env_state.if_tfstate_exists: url, kubeconfig = self._parse_config(env_state.build_dir) - current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" - self.state_service.set(DEFAULT, user=USER) - self._set_context_list(current_context) self._set_active_context(current_context) self.state_service.set(current_context) - self.state_service.set_section(current_context, BACKEND, url=url, token=auth_token) self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig) - self.state_service.write() return True, env_state @@ -177,14 +168,17 @@ def destroy(self, env_state, verbose=False): self.tf_service.cd_dir(env_state.build_dir) self.tf_service.set_verbose(verbose) - directory = self._tf_init(env_state.cloud, env_state.env, local_backend=False, destroying=True) + + self._tf_init(env_state, env_state.cloud, env_state.env, local_backend=False, destroying=True) + current_context = env_state.cloud if env_state.cloud in [DOCKER, MINIKUBE] \ else env_state.cloud + '_' + env_state.env + self._delete_resources(current_context) self._unset_context_list(current_context) self._remove_section(current_context) self._deactivate_context() - self.tf_service.destroy(directory, extra_vars) + self.tf_service.destroy(env_state.build_dir, extra_vars) self.tf_service.execute() self._remove_build_files(env_state.build_dir) self.state_service.write() @@ -207,25 +201,21 @@ def _delete_resources(self, context): if self.state_service.has_section(context, BACKEND): requests.delete(f"{self.url}/internal/resources") - def _tf_init(self, provider, env, local_backend, destroying=False): - directory = PROVIDER_DICT.get(provider) + def _tf_init(self, env_state, provider, env, local_backend, destroying=False): check_environment(provider) if is_cloud_provider(provider): - provider_directory = f"{directory}/{env}" - directory = f"{directory}/__working_{env}" - if not destroying or not os.path.isdir(directory): - copy_tree(provider_directory, directory) + if not destroying or not os.path.isdir(env_state.build_dir): + copy_tree(env_state.provider_directory, env_state.build_dir) if local_backend: - shutil.copy(LOCAL_CONFIG_DICT.get(provider), directory) + shutil.copy(LOCAL_CONFIG_DICT.get(provider), env_state.build_dir) # simply always create the workspace - self.tf_service.init(directory) - self.tf_service.new_workspace(directory, env) - self.tf_service.select_workspace(directory, env) - + self.tf_service.init(env_state.build_dir) + self.tf_service.new_workspace(env_state.build_dir, env) + self.tf_service.select_workspace(env_state.build_dir, env) else: - self.tf_service.init(directory) - return directory + copy_tree(env_state.provider_directory, env_state.build_dir) + self.tf_service.init(env_state.build_dir) def _set_context_list(self, current_context): try: @@ -236,17 +226,20 @@ def _set_context_list(self, current_context): updated_contexts = [] if isinstance(contexts, list): - contexts.append(current_context) + if current_context not in contexts: + contexts.append(current_context) updated_contexts = contexts elif isinstance(contexts, str) or not contexts: - # There is only one context or no context in available contexts - if contexts: - # exactly one available context - updated_contexts.append(contexts) - updated_contexts.append(current_context) - else: - # no available context - updated_contexts.append(current_context) + # check if current context is the same as existing context + if current_context != contexts: + # There is only one context or no context in available contexts + if contexts: + # exactly one available context + updated_contexts.append(contexts) + updated_contexts.append(current_context) + else: + # no available context + updated_contexts.append(current_context) self.state_service.set(CONTEXTS, environments=updated_contexts) @@ -311,7 +304,7 @@ def _parse_config(dir_build): @staticmethod def _get_vars(provider, dir_build, auth_token=None): - extra_vars = f"--var config_dir={dir_build} --var token={auth_token}" + extra_vars = f"--var config_dir={dir_build} --var token={auth_token} " if provider == AWS: KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") diff --git a/cli/kaos_cli/services/state_service.py b/cli/kaos_cli/services/state_service.py index 9afe325..3db960c 100644 --- a/cli/kaos_cli/services/state_service.py +++ b/cli/kaos_cli/services/state_service.py @@ -54,5 +54,7 @@ def full_delete(dir_build): shutil.rmtree(dir_build, ignore_errors=True) def write(self): + if not os.path.exists(KAOS_STATE_DIR): + os.mkdir(KAOS_STATE_DIR) with open(CONFIG_PATH, 'wb') as f: self.config.write(f) diff --git a/cli/kaos_cli/utils/validators.py b/cli/kaos_cli/utils/validators.py index d963d2f..2b2892e 100644 --- a/cli/kaos_cli/utils/validators.py +++ b/cli/kaos_cli/utils/validators.py @@ -3,12 +3,13 @@ import sys import socket from json import JSONDecodeError +import shutil import click import textdistance from kaos_cli.exceptions.exceptions import MissingArgumentError -from ..constants import KAOS_STATE_DIR, DOCKER, MINIKUBE, KAOS_TF_PATH, TF_STATE +from ..constants import KAOS_STATE_DIR, DOCKER, MINIKUBE, PROVIDER_DICT class EnvironmentState: @@ -20,6 +21,7 @@ def __init__(self): self.env = None self.build_dir = None self.tf_state_path = None + self.provider_directory = None self.if_build_dir_exists = None self.if_tfstate_exists = None @@ -28,17 +30,28 @@ def initialize(cls, cloud, env) -> "EnvironmentState": env_state = cls() env_state.cloud = cloud env_state.env = env - env_dir = f"{env_state.cloud}/{env_state.env}" if env_state.cloud not in [DOCKER, MINIKUBE] \ - else f"{env_state.cloud}" - build_dir = os.path.join(KAOS_TF_PATH, env_dir) - tf_state_path = os.path.join(KAOS_TF_PATH, env_dir, f"{TF_STATE}") + + infra_root = f"{PROVIDER_DICT.get(cloud)}" + + if is_cloud_provider(cloud): + provider_directory = f"{infra_root}/{env}" + build_dir = f"{infra_root}/__working_{env}" + tf_state_path = f"{build_dir}/.terraform/terraform.tfstate" + env_state.provider_directory = provider_directory + + else: + build_dir = f"{infra_root}" + tf_state_path = f"{build_dir}/terraform.tfstate" + env_state.provider_directory = infra_root + env_state.if_build_dir_exists = os.path.exists(build_dir) env_state.if_tfstate_exists = os.path.exists(tf_state_path) env_state.build_dir = build_dir env_state.tf_state_path = tf_state_path + return env_state - def validate_if_tfstate_exits(self) -> "EnvironmentState": + def validate_if_tf_state_exits(self) -> "EnvironmentState": """ Ensure existence of kaos backend dir (tf_path) and throw appropriate warnings if it does not exist """ @@ -76,6 +89,13 @@ def set_build_env(self) -> "EnvironmentState": else: self.env = 'prod' if not self.env else self.env # default = prod + def remove_terraform_files(self) -> "EnvironmentState": + """ + Simple method to remove existing terraform state for existing deployments + """ + tf_dir_path = f"{self.build_dir}/.terraform" + shutil.rmtree(tf_dir_path, ignore_errors=True) + def validate_index(n: int, ind: int, command: str): """ @@ -194,3 +214,7 @@ def validate_unused_port(port: int, host: str = '0.0.0.0') -> bool: except socket.error: return False + +def is_cloud_provider(cloud): + return cloud not in (DOCKER, MINIKUBE) + diff --git a/infrastructure/aws/envs/dev/main.tf b/infrastructure/aws/envs/dev/main.tf index c857fe2..e78b394 100644 --- a/infrastructure/aws/envs/dev/main.tf +++ b/infrastructure/aws/envs/dev/main.tf @@ -29,7 +29,6 @@ module "eks" { cluster_delete_timeout = "15m" kubeconfig_name = local.prefix config_output_path = "${path.module}/" - write_aws_auth_config = true workers_additional_policies = module.policies.worker_additional_policies tags = merge(local.tags, local.cluster_autoscaler_tags) diff --git a/infrastructure/aws/envs/prod/main.tf b/infrastructure/aws/envs/prod/main.tf index 458351c..6dbe2ef 100644 --- a/infrastructure/aws/envs/prod/main.tf +++ b/infrastructure/aws/envs/prod/main.tf @@ -29,7 +29,6 @@ module "eks" { cluster_delete_timeout = "15m" kubeconfig_name = local.prefix config_output_path = "${path.module}/" - write_aws_auth_config = true workers_additional_policies = module.policies.worker_additional_policies tags = merge(local.tags, local.cluster_autoscaler_tags) diff --git a/infrastructure/aws/envs/stage/main.tf b/infrastructure/aws/envs/stage/main.tf index 786dd13..b2c29e2 100644 --- a/infrastructure/aws/envs/stage/main.tf +++ b/infrastructure/aws/envs/stage/main.tf @@ -29,7 +29,6 @@ module "eks" { cluster_delete_timeout = "15m" kubeconfig_name = local.prefix config_output_path = "${path.module}/" - write_aws_auth_config = true workers_additional_policies = module.policies.worker_additional_policies tags = merge(local.tags, local.cluster_autoscaler_tags) diff --git a/infrastructure/gcp/modules/kubernetes/main.tf b/infrastructure/gcp/modules/kubernetes/main.tf index e1df9c2..8bb1f80 100644 --- a/infrastructure/gcp/modules/kubernetes/main.tf +++ b/infrastructure/gcp/modules/kubernetes/main.tf @@ -16,9 +16,9 @@ resource "google_container_cluster" "new_container_cluster" { disabled = var.http_load_balancing } - kubernetes_dashboard { - disabled = var.kubernetes_dashboard - } +// kubernetes_dashboard { +// disabled = var.kubernetes_dashboard +// } // network_policy_config { // disabled = "${var.network_policy_config}" // }