diff --git a/helpers/command.py b/helpers/command.py index 8063d033..4df343cc 100644 --- a/helpers/command.py +++ b/helpers/command.py @@ -25,11 +25,7 @@ def help(): ' -l, --logs', ' Display docker logs', ' -b, --build', - ' Build kpi and kobocat (only on dev/staging mode)', - ' -bkf, --build-kpi', - ' Build kpi (only on dev/staging mode)', - ' -bkc, --build-kobocat', - ' Build kobocat (only on dev/staging mode)', + ' Build django (kpi) container (only on dev/staging mode)', ' -s, --setup', ' Prompt questions to (re)write configuration files', ' -S, --stop', @@ -54,52 +50,27 @@ def help(): print('\n'.join(output)) @classmethod - def build(cls, image=None): + def build(cls): """ - Builds kpi/kobocat images with `--no-caches` option - Pulls latest `kobotoolbox/koboform_base` as well - - :param image: str + Builds kpi image with `--no-caches` option """ config = Config() dict_ = config.get_dict() if config.dev_mode or config.staging_mode: - def build_image(image_): - frontend_command = run_docker_compose(dict_, [ - '-f', 'docker-compose.frontend.yml', - '-f', 'docker-compose.frontend.override.yml', - '-p', config.get_prefix('frontend'), - 'build', '--force-rm', '--no-cache', - image_ - ]) - - CLI.run_command(frontend_command, dict_['kobodocker_path']) - - if image is None or image == 'kf': - prefix = config.get_prefix('frontend') - timestamp = int(time.time()) - dict_['kpi_dev_build_id'] = f'{prefix}{timestamp}' - config.write_config() - Template.render(config) - build_image('kpi') - - if image is None or image == 'kc': - pull_base_command = [ - 'docker', - 'pull', - 'kobotoolbox/koboform_base', - ] - - CLI.run_command(pull_base_command, dict_['kobodocker_path']) - - prefix = config.get_prefix('frontend') - timestamp = int(time.time()) - dict_['kc_dev_build_id'] = f'{prefix}{timestamp}' - config.write_config() - Template.render(config) - build_image('kobocat') + prefix = config.get_prefix('frontend') + timestamp = int(time.time()) + dict_['kpi_dev_build_id'] = f'{prefix}{timestamp}' + config.write_config() + Template.render(config) + frontend_command = run_docker_compose(dict_, [ + '-f', 'docker-compose.frontend.yml', + '-f', 'docker-compose.frontend.override.yml', + '-p', config.get_prefix('frontend'), + 'build', '--force-rm', '--no-cache', 'kpi' + ]) + CLI.run_command(frontend_command, dict_['kobodocker_path']) @classmethod def compose_frontend(cls, args): @@ -119,10 +90,9 @@ def compose_frontend(cls, args): def compose_backend(cls, args): config = Config() dict_ = config.get_dict() - backend_role = dict_['backend_server_role'] command = run_docker_compose(dict_, [ - '-f', f'docker-compose.backend.{backend_role}.yml', - '-f', f'docker-compose.backend.{backend_role}.override.yml', + '-f', f'docker-compose.backend.yml', + '-f', f'docker-compose.backend.override.yml', '-p', config.get_prefix('backend') ]) cls.__validate_custom_yml(config, command) @@ -163,8 +133,8 @@ def info(cls, timeout=600): elif int(time.time()) - start >= timeout: if timeout > 0: CLI.colored_print( - '\n`KoBoToolbox` has not started yet. ' - 'This is can be normal with low CPU/RAM computers.\n', + '\n`KoboToolbox` has not started yet. ' + 'This can happen with low CPU/RAM computers.\n', CLI.COLOR_INFO) question = f'Wait for another {timeout} seconds?' response = CLI.yes_no_question(question) @@ -209,7 +179,7 @@ def info(cls, timeout=600): else: message = ( - 'KoBoToolbox could not start!\n' + 'KoboToolbox could not start!\n' 'Please try `python3 run.py --logs` to see the logs.' ) CLI.framed_print(message, color=CLI.COLOR_ERROR) @@ -221,11 +191,10 @@ def logs(cls): config = Config() dict_ = config.get_dict() - if config.primary_backend or config.secondary_backend: - backend_role = dict_['backend_server_role'] + if config.backend: backend_command = run_docker_compose(dict_, [ - '-f', f'docker-compose.backend.{backend_role}.yml', - '-f', f'docker-compose.backend.{backend_role}.override.yml', + '-f', f'docker-compose.backend.yml', + '-f', f'docker-compose.backend.override.yml', '-p', config.get_prefix('backend'), 'logs', '-f' ]) @@ -310,13 +279,10 @@ def start(cls, frontend_only=False, force_setup=False): else: nginx_port = int(dict_['exposed_nginx_docker_port']) - if frontend_only or config.frontend or \ - not config.multi_servers: + if frontend_only or config.frontend or not config.multi_servers: ports.append(nginx_port) - if (not frontend_only or config.primary_backend or - config.secondary_backend) and \ - config.expose_backend_ports: + if not frontend_only and config.expose_backend_ports and config.backend: ports.append(dict_['postgresql_port']) ports.append(dict_['mongo_port']) ports.append(dict_['redis_main_port']) @@ -332,11 +298,9 @@ def start(cls, frontend_only=False, force_setup=False): # Start the back-end containers if not frontend_only and config.backend: - backend_role = dict_['backend_server_role'] - backend_command = run_docker_compose(dict_, [ - '-f', f'docker-compose.backend.{backend_role}.yml', - '-f', f'docker-compose.backend.{backend_role}.override.yml', + '-f', f'docker-compose.backend.yml', + '-f', f'docker-compose.backend.override.yml', '-p', config.get_prefix('backend'), 'up', '-d' ]) @@ -392,9 +356,8 @@ def start(cls, frontend_only=False, force_setup=False): 'It can take a few minutes.', CLI.COLOR_INFO) cls.info() else: - backend_server_role = dict_['backend_server_role'] CLI.colored_print( - (f'{backend_server_role} backend server is starting up ' + (f'Back-end server is starting up ' 'and should be up & running soon!\nPlease look at docker ' 'logs for further information: ' '`python3 run.py -cb logs -f`'), @@ -443,7 +406,6 @@ def stop_containers(cls, group: str, down: bool = False): config = Config() dict_ = config.get_dict() - backend_role = dict_['backend_server_role'] if group not in ['frontend', 'backend', 'certbot', 'maintenance']: raise Exception('Unknown group') @@ -459,8 +421,8 @@ def stop_containers(cls, group: str, down: bool = False): }, 'backend': { 'options': [ - '-f', f'docker-compose.backend.{backend_role}.yml', - '-f', f'docker-compose.backend.{backend_role}.override.yml', + '-f', f'docker-compose.backend.yml', + '-f', f'docker-compose.backend.override.yml', '-p', config.get_prefix('backend'), ], 'custom_yml': True, @@ -570,10 +532,8 @@ def __validate_custom_yml(config, command): command.insert(start_index + 1, 'docker-compose.frontend.custom.yml') if not frontend_command and dict_['use_backend_custom_yml']: - backend_server_role = dict_['backend_server_role'] - custom_file = '{}/docker-compose.backend.{}.custom.yml'.format( + custom_file = '{}/docker-compose.backend.custom.yml'.format( dict_['kobodocker_path'], - backend_server_role ) does_custom_file_exist = os.path.exists(custom_file) @@ -590,5 +550,5 @@ def __validate_custom_yml(config, command): command.insert(start_index, '-f') command.insert( start_index + 1, - 'docker-compose.backend.{}.custom.yml'.format(backend_server_role), + 'docker-compose.backend.custom.yml', ) diff --git a/helpers/config.py b/helpers/config.py index 5847f759..a9eeefe3 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -31,8 +31,8 @@ class Config(metaclass=Singleton): DEFAULT_PROXY_PORT = '8080' DEFAULT_NGINX_PORT = '80' DEFAULT_NGINX_HTTPS_PORT = '443' - KOBO_DOCKER_BRANCH = '2.024.17a' - KOBO_INSTALL_VERSION = '8.2.0' + KOBO_DOCKER_BRANCH = '2.024.25a' + KOBO_INSTALL_VERSION = '9.0.0' MAXIMUM_AWS_CREDENTIAL_ATTEMPTS = 3 ALLOWED_PASSWORD_CHARACTERS = ( string.ascii_letters @@ -81,7 +81,7 @@ def aws(self): @property def backend(self): - return not self.multi_servers or not self.frontend + return self.__dict['server_role'] == 'backend' or not self.multi_servers def build(self): """ @@ -113,7 +113,7 @@ def build(self): self.__questions_multi_servers() if self.multi_servers: self.__questions_roles() - if self.frontend or self.secondary_backend: + if self.frontend: self.__questions_private_routes() else: self.__reset(fake_dns=True) @@ -222,9 +222,6 @@ def get_upgraded_dict(self): # Upgrade to use two databases upgraded_dict = Upgrading.two_databases(upgraded_dict, self.__dict) - # Upgrade to use new terminology primary/secondary - upgraded_dict = Upgrading.new_terminology(upgraded_dict) - # Upgrade to use booleans in `self.__dict` upgraded_dict = Upgrading.use_booleans(upgraded_dict) @@ -313,8 +310,6 @@ def get_template(cls): 'aws_s3_region_name': 'us-east-1', 'aws_secret_key': '', 'aws_validate_credentials': True, - 'backend_server_role': 'primary', - 'backup_from_primary': True, 'block_common_http_ports': True, 'custom_secret_keys': False, 'customized_ports': False, @@ -337,7 +332,6 @@ def get_template(cls): 'https': True, 'internal_domain_name': 'docker.internal', 'kc_dev_build_id': '', - 'kc_path': '', 'kc_postgres_db': 'kobocat', 'kc_subdomain': 'kc', 'kobocat_media_backup_schedule': '0 0 * * 0', @@ -425,7 +419,6 @@ def get_template(cls): 'redis_main_port': '6379', 'redis_password': Config.generate_password(), 'review_host': True, - 'run_redis_containers': True, 'server_role': 'frontend', 'service_account_whitelisted_hosts': True, 'smtp_host': '', @@ -444,7 +437,6 @@ def get_template(cls): 'use_frontend_custom_yml': False, 'use_letsencrypt': True, 'use_private_dns': False, - 'use_wal_e': False, 'uwsgi_harakiri': '120', 'uwsgi_max_requests': '1024', 'uwsgi_settings': False, @@ -480,20 +472,6 @@ def local_install(self): def maintenance(self): self.__questions_maintenance() - @property - def primary_backend(self): - """ - Checks whether setup is running on a primary back-end server - - Returns: - bool - """ - return ( - self.multi_servers - and self.__dict['server_role'] == 'backend' - and self.__dict['backend_server_role'] == 'primary' - ) - @property def multi_servers(self): """ @@ -569,20 +547,6 @@ def read_unique_id(self): return unique_id - @property - def secondary_backend(self): - """ - Checks whether setup is running on a secondary back-end server - - Returns: - bool - """ - return ( - self.multi_servers - and self.__dict['server_role'] == 'backend' - and self.__dict['backend_server_role'] == 'secondary' - ) - def set_config(self, value): self.__dict = value @@ -655,22 +619,6 @@ def validate_passwords(self): CLI.COLOR_WARNING ) - # PostgreSQL replication password must be handled separately because - # it is set in PostgreSQL on the first launch and nothing is done - # afterwards in subsequent starts to update it if it has changed. - if not re.match( - psql_replication['pattern'], psql_replication['password'] - ): - CLI.colored_print( - 'PostgreSQL replication password contains unsupported characters.', - CLI.COLOR_ERROR - ) - CLI.colored_print( - 'It must be changed manually in `kobo-install/.run.conf` ' - '(and PostgreSQL itself if you use replication).', - CLI.COLOR_WARNING - ) - def write_config(self): """ Writes config to file `Config.CONFIG_FILE`. @@ -812,7 +760,8 @@ def __detect_network(self): choices.append('other') response = CLI.get_response( choices, - default=self.__dict['local_interface'] + default=self.__dict['local_interface'], + to_lower=False ) if response == 'other': @@ -954,8 +903,6 @@ def __questions_aws_backup_settings(self): if self.__dict['aws_backup_bucket_name'] != '': - backup_from_primary = self.__dict['backup_from_primary'] - CLI.colored_print('How many yearly backups to keep?', CLI.COLOR_QUESTION) self.__dict['aws_backup_yearly_retention'] = CLI.get_response( @@ -976,9 +923,7 @@ def __questions_aws_backup_settings(self): self.__dict['aws_backup_daily_retention'] = CLI.get_response( r'~^\d+$', self.__dict['aws_backup_daily_retention']) - if (not self.multi_servers or - (self.primary_backend and backup_from_primary) or - (self.secondary_backend and not backup_from_primary)): + if self.backend or not self.multi_servers: CLI.colored_print('PostgresSQL backup minimum size (in MB)?', CLI.COLOR_QUESTION) CLI.colored_print( @@ -990,7 +935,6 @@ def __questions_aws_backup_settings(self): r'~^\d+$', self.__dict['aws_postgres_backup_minimum_size']) - if self.primary_backend or not self.multi_servers: CLI.colored_print('MongoDB backup minimum size (in MB)?', CLI.COLOR_QUESTION) CLI.colored_print( @@ -1040,31 +984,12 @@ def __questions_backup(self): if self.backend and not self.frontend: self.__questions_aws() - # Prompting user whether they want to use WAL-E for - # continuous archiving - only if they are using aws - # for backups - if self.aws: - if self.primary_backend or not self.multi_servers: - self.__dict['use_wal_e'] = CLI.yes_no_question( - 'Do you want to use WAL-E for continuous ' - 'archiving of PostgreSQL backups?', - default=self.__dict['use_wal_e'] - ) - if self.__dict['use_wal_e']: - self.__dict['backup_from_primary'] = True - else: - # WAL-E cannot run on secondary - self.__dict['use_wal_e'] = False - else: - # WAL-E is only supported with AWS - self.__dict['use_wal_e'] = False - schedule_regex_pattern = ( r'^\-|((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))' r'(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})?$' ) message = ( - 'Schedules use linux cron syntax with UTC datetimes.\n' + 'Schedules use linux cron syntax with UTC date times.\n' 'For example, schedule at 12:00 AM E.S.T every Sunday ' 'would be:\n' '0 5 * * 0\n' @@ -1079,7 +1004,7 @@ def __questions_backup(self): color=CLI.COLOR_WARNING ) if self.frontend and not self.aws: - CLI.colored_print('KoBoCat media backup schedule?', + CLI.colored_print('KoboCat media backup schedule?', CLI.COLOR_QUESTION) self.__dict[ 'kobocat_media_backup_schedule'] = CLI.get_response( @@ -1087,60 +1012,34 @@ def __questions_backup(self): self.__dict['kobocat_media_backup_schedule']) if self.backend: - if self.__dict['use_wal_e'] or not self.multi_servers: - # We are on primary back-end server - self.__dict['backup_from_primary'] = True - backup_postgres = True - else: - if self.primary_backend: - default_response = self.__dict['backup_from_primary'] - else: - default_response = not self.__dict[ - 'backup_from_primary'] - - backup_postgres = CLI.yes_no_question( - 'Run PostgreSQL backup from this server?', - default=default_response - ) - - if self.primary_backend: - self.__dict['backup_from_primary'] = backup_postgres - else: - self.__dict['backup_from_primary'] = not backup_postgres - - if backup_postgres: - CLI.colored_print('PostgreSQL backup schedule?', - CLI.COLOR_QUESTION) - self.__dict[ - 'postgres_backup_schedule'] = CLI.get_response( + CLI.colored_print( + 'PostgreSQL backup schedule?', CLI.COLOR_QUESTION + ) + self.__dict['postgres_backup_schedule'] = ( + CLI.get_response( f'~{schedule_regex_pattern}', - self.__dict['postgres_backup_schedule']) - else: - self.__dict['postgres_backup_schedule'] = '' + self.__dict['postgres_backup_schedule'], + ) + ) - if self.primary_backend or not self.multi_servers: - CLI.colored_print('MongoDB backup schedule?', - CLI.COLOR_QUESTION) - self.__dict[ - 'mongo_backup_schedule'] = CLI.get_response( - f'~{schedule_regex_pattern}', - self.__dict['mongo_backup_schedule']) + CLI.colored_print( + 'MongoDB backup schedule?', CLI.COLOR_QUESTION + ) + self.__dict['mongo_backup_schedule'] = CLI.get_response( + f'~{schedule_regex_pattern}', + self.__dict['mongo_backup_schedule'], + ) - if self.__dict['run_redis_containers']: - CLI.colored_print('Redis backup schedule?', - CLI.COLOR_QUESTION) - self.__dict[ - 'redis_backup_schedule'] = CLI.get_response( - f'~{schedule_regex_pattern}', - self.__dict['redis_backup_schedule']) - else: - self.__dict['redis_backup_schedule'] = '' + CLI.colored_print( + 'Redis backup schedule?', CLI.COLOR_QUESTION + ) + self.__dict['redis_backup_schedule'] = CLI.get_response( + f'~{schedule_regex_pattern}', + self.__dict['redis_backup_schedule'], + ) if self.aws: self.__questions_aws_backup_settings() - else: - # Back to default value - self.__dict['backup_from_primary'] = True else: self.__reset(no_backups=True) else: @@ -1206,28 +1105,12 @@ def __questions_dev_mode(self): ) CLI.framed_print(message, color=CLI.COLOR_INFO) - kc_path = self.__dict['kc_path'] - self.__dict['kc_path'] = CLI.colored_input( - 'KoBoCat files location?', CLI.COLOR_QUESTION, - self.__dict['kc_path']) - self.__clone_repo(self.__dict['kc_path'], 'kobocat') - kpi_path = self.__dict['kpi_path'] self.__dict['kpi_path'] = CLI.colored_input( 'KPI files location?', CLI.COLOR_QUESTION, self.__dict['kpi_path']) self.__clone_repo(self.__dict['kpi_path'], 'kpi') - # Create an unique id to build fresh image - # when starting containers - if ( - not self.__dict['kc_dev_build_id'] or - self.__dict['kc_path'] != kc_path - ): - prefix = self.get_prefix('frontend') - timestamp = int(time.time()) - self.__dict['kc_dev_build_id'] = f'{prefix}{timestamp}' - if ( not self.__dict['kpi_dev_build_id'] or self.__dict['kpi_path'] != kpi_path @@ -1372,7 +1255,7 @@ def __questions_mongo(self): - primary back end - single server installation """ - if self.primary_backend or not self.multi_servers: + if self.backend or not self.multi_servers: mongo_user_username = self.__dict['mongo_user_username'] mongo_user_password = self.__dict['mongo_user_password'] mongo_root_username = self.__dict['mongo_root_username'] @@ -1479,7 +1362,7 @@ def __questions_postgres(self): Settings can be tweaked thanks to pgconfig.org API """ - CLI.colored_print('KoBoCat PostgreSQL database name?', + CLI.colored_print('KoboCat PostgreSQL database name?', CLI.COLOR_QUESTION) kc_postgres_db = CLI.get_response( r'~^\w+$', @@ -1792,7 +1675,7 @@ def __questions_public_routes(self): self.__dict['kpi_subdomain'] ) self.__dict['kc_subdomain'] = CLI.colored_input( - 'KoBoCat sub domain?', + 'KoboCat sub domain?', CLI.COLOR_QUESTION, self.__dict['kc_subdomain'] ) @@ -1817,18 +1700,14 @@ def __questions_raven(self): ) if self.__dict['raven_settings'] is True: self.__dict['kpi_raven'] = CLI.colored_input( - 'KPI Raven token', + 'KPI Sentry token', CLI.COLOR_QUESTION, self.__dict['kpi_raven']) - self.__dict['kobocat_raven'] = CLI.colored_input( - 'KoBoCat Raven token', CLI.COLOR_QUESTION, - self.__dict['kobocat_raven']) self.__dict['kpi_raven_js'] = CLI.colored_input( - 'KPI Raven JS token', CLI.COLOR_QUESTION, + 'KPI Sentry JS token', CLI.COLOR_QUESTION, self.__dict['kpi_raven_js']) else: self.__dict['kpi_raven'] = '' - self.__dict['kobocat_raven'] = '' self.__dict['kpi_raven_js'] = '' def __questions_redis(self): @@ -1837,48 +1716,39 @@ def __questions_redis(self): - primary back end - single server installation """ - if self.backend: - self.__dict['run_redis_containers'] = CLI.yes_no_question( - 'Do you want to run the Redis containers from this server?', - default=self.__dict['run_redis_containers'] + CLI.colored_print('Redis password?', CLI.COLOR_QUESTION) + self.__dict['redis_password'] = CLI.get_response( + self.__get_password_validation_pattern(allow_empty=True), + self.__dict['redis_password'], + to_lower=False, + error_msg=( + 'Invalid password. ' + 'Rules: Alphanumeric characters only, 8 characters minimum' ) - else: - self.__dict['run_redis_containers'] = True + ) - if self.__dict['run_redis_containers']: - CLI.colored_print('Redis password?', CLI.COLOR_QUESTION) - self.__dict['redis_password'] = CLI.get_response( - self.__get_password_validation_pattern(allow_empty=True), - self.__dict['redis_password'], - to_lower=False, - error_msg=( - 'Invalid password. ' - 'Rules: Alphanumeric characters only, 8 characters minimum' - ) + if not self.__dict['redis_password']: + message = ( + 'WARNING!\n\n' + 'It is STRONGLY recommended to set a password for Redis ' + 'as well.' ) + CLI.framed_print(message) + response = CLI.yes_no_question( + 'Do you want to continue without password?', + default=False + ) + if response is False: + self.__questions_redis() - if not self.__dict['redis_password']: - message = ( - 'WARNING!\n\n' - 'It is STRONGLY recommended to set a password for Redis ' - 'as well.' - ) - CLI.framed_print(message) - response = CLI.yes_no_question( - 'Do you want to continue without password?', - default=False - ) - if response is False: - self.__questions_redis() - - if self.backend: - CLI.colored_print('Max memory (MB) for Redis cache container?', - CLI.COLOR_QUESTION) - CLI.colored_print('Leave empty for no limits', - CLI.COLOR_INFO) - self.__dict['redis_cache_max_memory'] = CLI.get_response( - r'~^(\d+|-)?$', - self.__dict['redis_cache_max_memory']) + if self.backend: + CLI.colored_print( + 'Max memory (MB) for Redis cache container?', CLI.COLOR_QUESTION + ) + CLI.colored_print('Leave empty for no limits', CLI.COLOR_INFO) + self.__dict['redis_cache_max_memory'] = CLI.get_response( + r'~^(\d+|-)?$', self.__dict['redis_cache_max_memory'] + ) def __questions_reverse_proxy(self): @@ -1985,26 +1855,15 @@ def __questions_reverse_proxy(self): self.__dict['block_common_http_ports'] = False def __questions_roles(self): - CLI.colored_print('Which role do you want to assign to this server?', - CLI.COLOR_QUESTION) + CLI.colored_print( + 'Which role do you want to assign to this server?', + CLI.COLOR_QUESTION, + ) CLI.colored_print('\t1) frontend') CLI.colored_print('\t2) backend') self.__dict['server_role'] = CLI.get_response( - ['backend', 'frontend'], - self.__dict['server_role']) - - if self.__dict['server_role'] == 'backend': - CLI.colored_print( - 'Which role do you want to assign to this back-end server?', - CLI.COLOR_QUESTION) - CLI.colored_print('\t1) primary') - CLI.colored_print('\t2) secondary') - self.__dict['backend_server_role'] = CLI.get_response( - ['primary', 'secondary'], - self.__dict['backend_server_role']) - else: - # It may be useless to force back-end role when using multi servers. - self.__dict['backend_server_role'] = 'primary' + ['backend', 'frontend'], self.__dict['server_role'] + ) def __questions_secret_keys(self): self.__dict['custom_secret_keys'] = CLI.yes_no_question( @@ -2044,10 +1903,12 @@ def __questions_secret_keys(self): def __questions_service_account(self): if not self.local_install: - self.__dict['service_account_whitelisted_hosts'] = CLI.yes_no_question( - 'Do you want to restrict API calls between KPI and KoBoCAT ' - 'to their internal domain names?', - default=self.__dict['service_account_whitelisted_hosts'] + self.__dict['service_account_whitelisted_hosts'] = ( + CLI.yes_no_question( + 'Do you want to restrict API calls between KPI and KoboCAT ' + 'to their internal domain names?', + default=self.__dict['service_account_whitelisted_hosts'], + ) ) else: self.__dict['service_account_whitelisted_hosts'] = False @@ -2072,15 +1933,15 @@ def __questions_session_cookies(self): ) def __questions_smtp(self): - self.__dict['smtp_host'] = CLI.colored_input('SMTP server?', - CLI.COLOR_QUESTION, - self.__dict['smtp_host']) - self.__dict['smtp_port'] = CLI.colored_input('SMTP port?', - CLI.COLOR_QUESTION, - self.__dict['smtp_port']) - self.__dict['smtp_user'] = CLI.colored_input('SMTP user?', - CLI.COLOR_QUESTION, - self.__dict['smtp_user']) + self.__dict['smtp_host'] = CLI.colored_input( + 'SMTP server?', CLI.COLOR_QUESTION, self.__dict['smtp_host'] + ) + self.__dict['smtp_port'] = CLI.colored_input( + 'SMTP port?', CLI.COLOR_QUESTION, self.__dict['smtp_port'] + ) + self.__dict['smtp_user'] = CLI.colored_input( + 'SMTP user?', CLI.COLOR_QUESTION, self.__dict['smtp_user'] + ) if self.__dict['smtp_user']: self.__dict['smtp_password'] = CLI.colored_input( 'SMTP password', @@ -2213,7 +2074,6 @@ def __reset(self, **kwargs): if production or all_: self.__dict['dev_mode'] = False self.__dict['staging_mode'] = False - self.__dict['kc_path'] = '' self.__dict['kpi_path'] = '' self.__dict['debug'] = False self.__dict['use_celery'] = True @@ -2232,7 +2092,6 @@ def __reset(self, **kwargs): self.__dict['use_letsencrypt'] = False if no_backups or all_: - self.__dict['backup_from_primary'] = True self.__dict['use_backup'] = False self.__dict['use_wal_e'] = False @@ -2280,11 +2139,11 @@ def __validate_installation(self): 'You are installing over existing data.\n' '\n' 'It is recommended to backup your data and import it ' - 'to a fresh installed (by KoBoInstall) database.\n' + 'to a fresh installed (by kobo-install) database.\n' '\n' 'kobo-install uses these images:\n' - ' - MongoDB: mongo:3.4\n' - ' - PostgreSQL: mdillon/postgis:9.5\n' + ' - MongoDB: mongo:5.0\n' + ' - PostgreSQL: postgis/postgis:14-3.2\n' '\n' 'Be sure to upgrade to these versions before going ' 'further!' diff --git a/helpers/template.py b/helpers/template.py index 08141520..34bfd2c4 100644 --- a/helpers/template.py +++ b/helpers/template.py @@ -156,7 +156,7 @@ def _get_value(property_, true_value='', false_value='#', nginx_port = dict_['exposed_nginx_docker_port'] return { - 'PROTOCOL': _get_value('https', 'https', 'http'), + 'PUBLIC_REQUEST_SCHEME': _get_value('https', 'https', 'http'), 'USE_HTTPS': _get_value('https'), 'USE_AWS': _get_value('use_aws'), 'AWS_ACCESS_KEY_ID': dict_['aws_access_key'], @@ -178,7 +178,8 @@ def _get_value(property_, true_value='', false_value='#', 'DJANGO_SESSION_COOKIE_AGE': dict_['django_session_cookie_age'], 'ENKETO_ENCRYPTION_KEY': dict_['enketo_encryption_key'], 'ENKETO_LESS_SECURE_ENCRYPTION_KEY': dict_[ - 'enketo_less_secure_encryption_key'], + 'enketo_less_secure_encryption_key' + ], 'KOBOCAT_RAVEN_DSN': dict_['kobocat_raven'], 'KPI_RAVEN_DSN': dict_['kpi_raven'], 'KPI_RAVEN_JS_DSN': dict_['kpi_raven_js'], @@ -195,46 +196,40 @@ def _get_value(property_, true_value='', false_value='#', 'DEFAULT_FROM_EMAIL': dict_['default_from_email'], 'PRIMARY_BACKEND_IP': dict_['primary_backend_ip'], 'LOCAL_INTERFACE_IP': dict_['local_interface_ip'], - 'KC_PATH': dict_['kc_path'], 'KPI_PATH': dict_['kpi_path'], - 'USE_KPI_DEV_MODE': _get_value('kpi_path', - true_value='#', - false_value='', - comparison_value=''), - 'USE_KC_DEV_MODE': _get_value('kc_path', - true_value='#', - false_value='', - comparison_value=''), - 'KC_DEV_BUILD_ID': dict_['kc_dev_build_id'], + 'USE_KPI_DEV_MODE': _get_value( + 'kpi_path', true_value='#', false_value='', comparison_value='' + ), 'KPI_DEV_BUILD_ID': dict_['kpi_dev_build_id'], - 'NGINX_PUBLIC_PORT': dict_['exposed_nginx_docker_port'], + 'NGINX_PUBLIC_PORT': ( + '' + if dict_['exposed_nginx_docker_port'] == '80' + else f":{dict_['exposed_nginx_docker_port']}" + ), 'NGINX_EXPOSED_PORT': nginx_port, 'UWSGI_WORKERS_MAX': dict_['uwsgi_workers_max'], # Deactivate cheaper algorithm if defaults are 1 worker to start and # 2 maximum. 'UWSGI_WORKERS_START': ( '' - if dict_['uwsgi_workers_start'] == '1' and dict_['uwsgi_workers_max'] == '2' + if dict_['uwsgi_workers_start'] == '1' + and dict_['uwsgi_workers_max'] == '2' else dict_['uwsgi_workers_start'] ), 'UWSGI_MAX_REQUESTS': dict_['uwsgi_max_requests'], - 'UWSGI_SOFT_LIMIT': int( - dict_['uwsgi_soft_limit']) * 1024 * 1024, + 'UWSGI_SOFT_LIMIT': int(dict_['uwsgi_soft_limit']) * 1024 * 1024, 'UWSGI_HARAKIRI': dict_['uwsgi_harakiri'], - 'UWSGI_WORKER_RELOAD_MERCY': dict_[ - 'uwsgi_worker_reload_mercy'], + 'UWSGI_WORKER_RELOAD_MERCY': dict_['uwsgi_worker_reload_mercy'], 'UWSGI_PASS_TIMEOUT': int(dict_['uwsgi_harakiri']) + 10, 'POSTGRES_REPLICATION_PASSWORD': dict_[ - 'postgres_replication_password'], - 'WSGI_SERVER': 'runserver_plus' if config.dev_mode else 'uWSGI', + 'postgres_replication_password' + ], + 'WSGI': 'runserver_plus' if config.dev_mode else 'uWSGI', 'USE_X_FORWARDED_HOST': '' if config.dev_mode else '#', 'OVERRIDE_POSTGRES_SETTINGS': _get_value('postgres_settings'), 'POSTGRES_APP_PROFILE': dict_['postgres_profile'], 'POSTGRES_RAM': dict_['postgres_ram'], 'POSTGRES_SETTINGS': dict_['postgres_settings_content'], - 'POSTGRES_BACKUP_FROM_SECONDARY': _get_value( - 'backup_from_primary', - comparison_value=False), 'POSTGRES_PORT': dict_['postgresql_port'], 'MONGO_PORT': dict_['mongo_port'], 'REDIS_MAIN_PORT': dict_['redis_main_port'], @@ -242,69 +237,86 @@ def _get_value(property_, true_value='', false_value='#', 'REDIS_CACHE_MAX_MEMORY': dict_['redis_cache_max_memory'], 'USE_BACKUP': '' if dict_['use_backup'] else '#', 'USE_WAL_E': _get_value('use_wal_e'), - 'USE_AWS_BACKUP': '' if (config.aws and - dict_['aws_backup_bucket_name'] != '' and - dict_['use_backup']) else '#', - 'USE_MEDIA_BACKUP': '' if (not config.aws and - dict_['use_backup']) else '#', + 'USE_AWS_BACKUP': ( + '' + if ( + config.aws + and dict_['aws_backup_bucket_name'] != '' + and dict_['use_backup'] + ) + else '#' + ), + 'USE_MEDIA_BACKUP': ( + '' if (not config.aws and dict_['use_backup']) else '#' + ), 'KOBOCAT_MEDIA_BACKUP_SCHEDULE': dict_[ - 'kobocat_media_backup_schedule'], + 'kobocat_media_backup_schedule' + ], 'MONGO_BACKUP_SCHEDULE': dict_['mongo_backup_schedule'], 'POSTGRES_BACKUP_SCHEDULE': dict_['postgres_backup_schedule'], 'REDIS_BACKUP_SCHEDULE': dict_['redis_backup_schedule'], 'AWS_BACKUP_BUCKET_NAME': dict_['aws_backup_bucket_name'], - 'AWS_BACKUP_YEARLY_RETENTION': dict_[ - 'aws_backup_yearly_retention'], + 'AWS_BACKUP_YEARLY_RETENTION': dict_['aws_backup_yearly_retention'], 'AWS_BACKUP_MONTHLY_RETENTION': dict_[ - 'aws_backup_monthly_retention'], - 'AWS_BACKUP_WEEKLY_RETENTION': dict_[ - 'aws_backup_weekly_retention'], - 'AWS_BACKUP_DAILY_RETENTION': dict_[ - 'aws_backup_daily_retention'], + 'aws_backup_monthly_retention' + ], + 'AWS_BACKUP_WEEKLY_RETENTION': dict_['aws_backup_weekly_retention'], + 'AWS_BACKUP_DAILY_RETENTION': dict_['aws_backup_daily_retention'], 'AWS_MONGO_BACKUP_MINIMUM_SIZE': dict_[ - 'aws_mongo_backup_minimum_size'], + 'aws_mongo_backup_minimum_size' + ], 'AWS_POSTGRES_BACKUP_MINIMUM_SIZE': dict_[ - 'aws_postgres_backup_minimum_size'], + 'aws_postgres_backup_minimum_size' + ], 'AWS_REDIS_BACKUP_MINIMUM_SIZE': dict_[ - 'aws_redis_backup_minimum_size'], + 'aws_redis_backup_minimum_size' + ], 'AWS_BACKUP_UPLOAD_CHUNK_SIZE': dict_[ - 'aws_backup_upload_chunk_size'], + 'aws_backup_upload_chunk_size' + ], 'AWS_BACKUP_BUCKET_DELETION_RULE_ENABLED': _get_value( - 'aws_backup_bucket_deletion_rule_enabled', 'True', 'False'), + 'aws_backup_bucket_deletion_rule_enabled', 'True', 'False' + ), 'LETSENCRYPT_EMAIL': dict_['letsencrypt_email'], 'MAINTENANCE_ETA': dict_['maintenance_eta'], 'MAINTENANCE_DATE_ISO': dict_['maintenance_date_iso'], 'MAINTENANCE_DATE_STR': dict_['maintenance_date_str'], 'MAINTENANCE_EMAIL': dict_['maintenance_email'], - 'USE_NPM_FROM_HOST': '' if (config.dev_mode and - not dict_['npm_container']) else '#', + 'USE_NPM_FROM_HOST': ( + '' if (config.dev_mode and not dict_['npm_container']) else '#' + ), 'DOCKER_NETWORK_BACKEND_PREFIX': config.get_prefix('backend'), 'DOCKER_NETWORK_FRONTEND_PREFIX': config.get_prefix('frontend'), - 'USE_BACKEND_NETWORK': _get_value('expose_backend_ports', - comparison_value=False), + 'USE_BACKEND_NETWORK': _get_value( + 'expose_backend_ports', comparison_value=False + ), 'EXPOSE_BACKEND_PORTS': _get_value('expose_backend_ports'), 'USE_FAKE_DNS': _get_value('local_installation'), - 'ADD_BACKEND_EXTRA_HOSTS': '' if ( - config.expose_backend_ports and - not config.use_private_dns) else '#', - 'USE_EXTRA_HOSTS': '' if (config.local_install or - config.expose_backend_ports and - not config.use_private_dns) else '#', + 'ADD_BACKEND_EXTRA_HOSTS': ( + '' + if (config.expose_backend_ports and not config.use_private_dns) + else '#' + ), + 'USE_EXTRA_HOSTS': ( + '' + if ( + config.local_install + or config.expose_backend_ports + and not config.use_private_dns + ) + else '#' + ), 'MONGO_ROOT_USERNAME': dict_['mongo_root_username'], 'MONGO_ROOT_PASSWORD': dict_['mongo_root_password'], 'MONGO_USER_USERNAME': dict_['mongo_user_username'], 'MONGO_USER_PASSWORD': dict_['mongo_user_password'], 'REDIS_PASSWORD': dict_['redis_password'], - 'REDIS_PASSWORD_JS_ENCODED': json.dumps( - dict_['redis_password']), + 'REDIS_PASSWORD_JS_ENCODED': json.dumps(dict_['redis_password']), 'USE_DEV_MODE': _get_value('dev_mode'), 'USE_CELERY': _get_value('use_celery', comparison_value=False), 'ENKETO_ALLOW_PRIVATE_IP_ADDRESS': _get_value( - 'local_installation', - true_value='true', - false_value='false' + 'local_installation', true_value='true', false_value='false' ), - 'RUN_REDIS_CONTAINERS': _get_value('run_redis_containers'), 'USE_REDIS_CACHE_MAX_MEMORY': _get_value( 'redis_cache_max_memory', true_value='#', @@ -323,7 +335,7 @@ def _get_value(property_, true_value='', false_value='#', # Keep leading space in front of suffix if any 'DOCKER_COMPOSE_SUFFIX': _get_value( 'compose_version', '', 'compose', 'v1' - ) + ), } @staticmethod diff --git a/helpers/upgrading.py b/helpers/upgrading.py index 3b58a068..2ae51805 100644 --- a/helpers/upgrading.py +++ b/helpers/upgrading.py @@ -22,7 +22,6 @@ def migrate_single_to_two_databases(config: 'helpers.Config'): config (helpers.config.Config) """ dict_ = config.get_dict() - backend_role = dict_['backend_server_role'] def _kpi_db_alias_kludge(command): """ @@ -73,7 +72,7 @@ def _kpi_db_alias_kludge(command): ) message = ( 'Upgrading to separate databases is required to run the latest ' - 'release of KoBoToolbox, but it may be a slow process if you ' + 'release of KoboToolbox, but it may be a slow process if you ' 'have a lot of data. Expect at least one minute of downtime ' 'for every 1,500 KPI assets. Assets are surveys and library ' 'items: questions, blocks, and templates.\n' @@ -106,7 +105,7 @@ def _kpi_db_alias_kludge(command): '-f', f'docker-compose.backend.{backend_role}.override.yml', '-p', config.get_prefix('backend'), 'exec', 'postgres', 'bash', - '/kobo-docker-scripts/primary/clone_data_from_kc_to_kpi.sh', + '/kobo-docker-scripts/scripts/clone_data_from_kc_to_kpi.sh', '--noinput' ]) try: @@ -127,26 +126,6 @@ def _kpi_db_alias_kludge(command): sys.stderr.write(kpi_kc_db_empty) sys.exit(1) - @staticmethod - def new_terminology(upgraded_dict: dict) -> dict: - """ - Updates configuration to use new `kobo-docker` terminology. - See: https://github.com/kobotoolbox/kobo-docker/pull/294 - - Args: - upgraded_dict (dict): Configuration values to be upgraded - - Returns: - dict - """ - - backend_role = upgraded_dict['backend_server_role'] - if backend_role in ['master', 'slave']: - upgraded_dict['backend_server_role'] = 'primary' \ - if backend_role == 'master' else 'secondary' - - return upgraded_dict - @staticmethod def set_compose_version(upgraded_dict: dict) -> dict: diff --git a/readme.md b/readme.md index 367d4edb..ecb0f8ba 100644 --- a/readme.md +++ b/readme.md @@ -43,13 +43,13 @@ Get info: Get docker logs: `$kobo-install> python3 run.py --logs` -Update KoBoToolbox: +Update KoboToolbox: `$kobo-install> python3 run.py --update [branch or tag]` By default, fetch the latest version of `master` branch -Stop KoBoToolbox: +Stop KoboToolbox: `$kobo-install> python3 run.py --stop` Get help: @@ -76,8 +76,8 @@ Stop maintenance mode: ## Build the configuration User can choose between 2 types of installations: -- `Workstation`: KoBoToolbox doesn't need to be accessible from anywhere except the computer where it's installed. No DNS needed -- `Server`: KoBoToolbox needs to be accessible from the local network or from the Internet. DNS are needed +- `Workstation`: KoboToolbox doesn't need to be accessible from anywhere except the computer where it's installed. No DNS needed +- `Server`: KoboToolbox needs to be accessible from the local network or from the Internet. DNS are needed ### Options @@ -94,38 +94,37 @@ User can choose between 2 types of installations: ### Advanced Options -|Option|Default|Workstation|Server -|---|---|---|---| -|Webserver port| **80** | ✓ | | -|Reverse proxy interal port| **8080** | | ✓ (front end only) | -|Network interface| **Autodetected** | ✓ | ✓ (front end only) | -|Use separate servers| **No** | | ✓ | -|Use DNS for private routes| **No** | | ✓ (front end only) | -|Primary back end IP _(if previous answer is no)_| **Local IP** | | ✓ (front end only) | -|PostgreSQL DB| **kobo** | ✓ | ✓ | -|PostgreSQL user's username| **kobo** | ✓ | ✓ | -|PostgreSQL user's password| **Autogenerate** | ✓ | ✓ | -|PostgreSQL number of connections3| **100** | ✓ | ✓ (back end only) | -|PostgreSQL RAM3| **2** | ✓ | ✓ (back end only) | -|PostgreSQL Application Profile3| **Mixed** | ✓ | ✓ (back end only) | -|PostgreSQL Storage3| **HDD** | ✓ | ✓ (back end only) | -|MongoDB super user's username| **root** | ✓ | ✓ | -|MongoDB super user's password| **Autogenerate** | ✓ | ✓ | -|MongoDB user's username| **kobo** | ✓ | ✓ | -|MongoDB user's password| **Autogenerate** | ✓ | ✓ | -|Redis password4| **Autogenerate** | ✓ | ✓ | -|Use AWS storage5| **No** | ✓ | ✓ | -|Use WAL-E PostgreSQL backups6 | **No** | ✓ | ✓ (back end only) | -|uWGI workers| **start: 2, max: 4** | ✓ | ✓ (front end only) | -|uWGI memory limit| **128 MB** | ✓ | ✓ (front end only) | -|uWGI harakiri timeout | **120s** | ✓ | ✓ (front end only) | -|uWGI worker reload timeout | **120s** | ✓ | ✓ (front end only) | -|Google UA| | ✓ | ✓ (front end only) | -|Google API Key| | ✓ | ✓ (front end only) | -|Raven tokens| | ✓ | ✓ (front end only) | -|Debug| **False** | ✓ | | -|Developer mode| **False** | ✓ | | -|Staging mode| **False** | | ✓ (front end only) | +| Option |Default|Workstation|Server +|-------------------------------------------------|---|---|---| +| Webserver port | **80** | ✓ | | +| Reverse proxy interal port | **8080** | | ✓ (front end only) | +| Network interface | **Autodetected** | ✓ | ✓ (front end only) | +| Use separate servers | **No** | | ✓ | +| Use DNS for private routes | **No** | | ✓ (front end only) | +| Back-end server IP _(if previous answer is no)_ | **Local IP** | | ✓ (front end only) | +| PostgreSQL DB | **kobo** | ✓ | ✓ | +| PostgreSQL user's username | **kobo** | ✓ | ✓ | +| PostgreSQL user's password | **Autogenerate** | ✓ | ✓ | +| PostgreSQL number of connections3 | **100** | ✓ | ✓ (back end only) | +| PostgreSQL RAM3 | **2** | ✓ | ✓ (back end only) | +| PostgreSQL Application Profile3 | **Mixed** | ✓ | ✓ (back end only) | +| PostgreSQL Storage3 | **HDD** | ✓ | ✓ (back end only) | +| MongoDB super user's username | **root** | ✓ | ✓ | +| MongoDB super user's password | **Autogenerate** | ✓ | ✓ | +| MongoDB user's username | **kobo** | ✓ | ✓ | +| MongoDB user's password | **Autogenerate** | ✓ | ✓ | +| Redis password4 | **Autogenerate** | ✓ | ✓ | +| Use AWS storage5 | **No** | ✓ | ✓ | +| uWGI workers | **start: 2, max: 4** | ✓ | ✓ (front end only) | +| uWGI memory limit | **128 MB** | ✓ | ✓ (front end only) | +| uWGI harakiri timeout | **120s** | ✓ | ✓ (front end only) | +| uWGI worker reload timeout | **120s** | ✓ | ✓ (front end only) | +| Google UA | | ✓ | ✓ (front end only) | +| Google API Key | | ✓ | ✓ (front end only) | +| Sentry tokens | | ✓ | ✓ (front end only) | +| Debug | **False** | ✓ | | +| Developer mode | **False** | ✓ | | +| Staging mode | **False** | | ✓ (front end only) | 1) _HTTPS certificates must be installed on a Reverse Proxy. `kobo-install` can install one and use `Let's Encrypt` to generate certificates @@ -140,8 +139,6 @@ User can choose between 2 types of installations: 5) _If AWS storage is selected, credentials must be provided if backups are activated_ -6) _WAL-E backups for PostgreSQL are only available when using AWS storage_ - ℹ Intercom App ID [must now](https://github.com/kobotoolbox/kpi/pull/2285) be configured through "Per user settings" in the Django admin interface of KPI. ## Requirements diff --git a/run.py b/run.py index 1a84ce6c..3cd39eb5 100755 --- a/run.py +++ b/run.py @@ -98,10 +98,6 @@ def run(force_setup=False): Command.logs() elif sys.argv[1] == '-b' or sys.argv[1] == '--build': Command.build() - elif sys.argv[1] == '-bkf' or sys.argv[1] == '--build-kpi': - Command.build('kf') - elif sys.argv[1] == '-bkc' or sys.argv[1] == '--build-kobocat': - Command.build('kc') elif sys.argv[1] == '-v' or sys.argv[1] == '--version': Command.version() elif sys.argv[1] == '-m' or sys.argv[1] == '--maintenance': diff --git a/templates/kobo-docker/docker-compose.backend.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.override.yml.tpl new file mode 100644 index 00000000..21f2fe2d --- /dev/null +++ b/templates/kobo-docker/docker-compose.backend.override.yml.tpl @@ -0,0 +1,42 @@ +# Override for primary back-end server +version: '3' + +services: + + postgres: + volumes: + - ../kobo-env/postgres/conf/postgres.conf:/kobo-docker-scripts/conf/postgres.conf + ${EXPOSE_BACKEND_PORTS}ports: + ${EXPOSE_BACKEND_PORTS} - ${POSTGRES_PORT}:5432 + ${USE_BACKEND_NETWORK}networks: + ${USE_BACKEND_NETWORK} kobo-be-network: + ${USE_BACKEND_NETWORK} aliases: + ${USE_BACKEND_NETWORK} - postgres.${PRIVATE_DOMAIN_NAME} + + mongo: + ${EXPOSE_BACKEND_PORTS}ports: + ${EXPOSE_BACKEND_PORTS} - ${MONGO_PORT}:27017 + ${USE_BACKEND_NETWORK}networks: + ${USE_BACKEND_NETWORK} kobo-be-network: + ${USE_BACKEND_NETWORK} aliases: + ${USE_BACKEND_NETWORK} - mongo.${PRIVATE_DOMAIN_NAME} + + redis_main: + ${EXPOSE_BACKEND_PORTS}ports: + ${EXPOSE_BACKEND_PORTS} - ${REDIS_MAIN_PORT}:6379 + ${USE_BACKEND_NETWORK}networks: + ${USE_BACKEND_NETWORK} kobo-be-network: + ${USE_BACKEND_NETWORK} aliases: + ${USE_BACKEND_NETWORK} - redis-main.${PRIVATE_DOMAIN_NAME} + + redis_cache: + ${EXPOSE_BACKEND_PORTS}ports: + ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 + ${USE_BACKEND_NETWORK}networks: + ${USE_BACKEND_NETWORK} kobo-be-network: + ${USE_BACKEND_NETWORK} aliases: + ${USE_BACKEND_NETWORK} - redis-cache.${PRIVATE_DOMAIN_NAME} + +${USE_BACKEND_NETWORK}networks: +${USE_BACKEND_NETWORK} kobo-be-network: +${USE_BACKEND_NETWORK} driver: bridge diff --git a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl deleted file mode 100644 index 72d37da6..00000000 --- a/templates/kobo-docker/docker-compose.backend.primary.override.yml.tpl +++ /dev/null @@ -1,50 +0,0 @@ -# Override for primary back-end server -version: '2.2' - -services: - - postgres: - volumes: - - ../kobo-env/postgres/primary/postgres.conf:/kobo-docker-scripts/primary/postgres.conf - ${POSTGRES_BACKUP_FROM_SECONDARY}environment: - ${POSTGRES_BACKUP_FROM_SECONDARY} - POSTGRES_BACKUP_FROM_SECONDARY=True - ${EXPOSE_BACKEND_PORTS}ports: - ${EXPOSE_BACKEND_PORTS} - ${POSTGRES_PORT}:5432 - ${USE_BACKEND_NETWORK}networks: - ${USE_BACKEND_NETWORK} kobo-be-network: - ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - postgres.${PRIVATE_DOMAIN_NAME} - - mongo: - ${EXPOSE_BACKEND_PORTS}ports: - ${EXPOSE_BACKEND_PORTS} - ${MONGO_PORT}:27017 - ${USE_BACKEND_NETWORK}networks: - ${USE_BACKEND_NETWORK} kobo-be-network: - ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - mongo.${PRIVATE_DOMAIN_NAME} - - ${RUN_REDIS_CONTAINERS}redis_main: - ${RUN_REDIS_CONTAINERS} extends: - ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml - ${RUN_REDIS_CONTAINERS} service: redis_main - ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: - ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_MAIN_PORT}:6379 - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-main.${PRIVATE_DOMAIN_NAME} - - ${RUN_REDIS_CONTAINERS}redis_cache: - ${RUN_REDIS_CONTAINERS} extends: - ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml - ${RUN_REDIS_CONTAINERS} service: redis_cache - ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS}ports: - ${RUN_REDIS_CONTAINERS} ${EXPOSE_BACKEND_PORTS} - ${REDIS_CACHE_PORT}:6380 - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK}networks: - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} kobo-be-network: - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} aliases: - ${RUN_REDIS_CONTAINERS} ${USE_BACKEND_NETWORK} - redis-cache.${PRIVATE_DOMAIN_NAME} - -${USE_BACKEND_NETWORK}networks: -${USE_BACKEND_NETWORK} kobo-be-network: -${USE_BACKEND_NETWORK} driver: bridge diff --git a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl b/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl deleted file mode 100644 index 2fb79929..00000000 --- a/templates/kobo-docker/docker-compose.backend.secondary.override.yml.tpl +++ /dev/null @@ -1,27 +0,0 @@ -# For public, HTTPS servers. -version: '2.2' - -services: - postgres: - volumes: - - ../kobo-env/postgres/secondary/postgres.conf:/kobo-docker-scripts/secondary/postgres.conf - ports: - - ${POSTGRES_PORT}:5432 - ${ADD_BACKEND_EXTRA_HOSTS}extra_hosts: - ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - ${ADD_BACKEND_EXTRA_HOSTS} - primary.postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} - - ${RUN_REDIS_CONTAINERS}redis_main: - ${RUN_REDIS_CONTAINERS} extends: - ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml - ${RUN_REDIS_CONTAINERS} service: redis_main - ${RUN_REDIS_CONTAINERS} ports: - ${RUN_REDIS_CONTAINERS} - ${REDIS_MAIN_PORT}:6379 - - ${RUN_REDIS_CONTAINERS}redis_cache: - ${RUN_REDIS_CONTAINERS} extends: - ${RUN_REDIS_CONTAINERS} file: docker-compose.backend.template.yml - ${RUN_REDIS_CONTAINERS} service: redis_cache - ${RUN_REDIS_CONTAINERS} ports: - ${RUN_REDIS_CONTAINERS} - ${REDIS_CACHE_PORT}:6380 - diff --git a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl index 672d3631..e81d791f 100644 --- a/templates/kobo-docker/docker-compose.frontend.override.yml.tpl +++ b/templates/kobo-docker/docker-compose.frontend.override.yml.tpl @@ -2,22 +2,23 @@ version: '3' services: - kobocat: - ${USE_KC_DEV_MODE} build: ${KC_PATH} - ${USE_KC_DEV_MODE} image: kobocat:dev.${KC_DEV_BUILD_ID} - ${USE_KC_DEV_MODE} volumes: - ${USE_KC_DEV_MODE} - ${KC_PATH}:/srv/src/kobocat + kpi: + ${USE_KPI_DEV_MODE} build: ${KPI_PATH} + ${USE_KPI_DEV_MODE} image: kpi:dev.${KPI_DEV_BUILD_ID} + ${USE_KPI_DEV_MODE} volumes: + ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi environment: - - ENKETO_PROTOCOL=${PROTOCOL} - - KC_UWSGI_WORKERS_COUNT=${UWSGI_WORKERS_MAX} - - KC_UWSGI_CHEAPER_WORKERS_COUNT=${UWSGI_WORKERS_START} - - NGINX_PUBLIC_PORT=${NGINX_PUBLIC_PORT} - - KC_UWSGI_MAX_REQUESTS=${UWSGI_MAX_REQUESTS} - - KC_UWSGI_CHEAPER_RSS_LIMIT_SOFT=${UWSGI_SOFT_LIMIT} - - KC_UWSGI_HARAKIRI=${UWSGI_HARAKIRI} - - KC_UWSGI_WORKER_RELOAD_MERCY=${UWSGI_WORKER_RELOAD_MERCY} - ${USE_DEV_MODE}- DJANGO_SETTINGS_MODULE=onadata.settings.dev - ${USE_CELERY}- SKIP_CELERY=True + - UWSGI_WORKERS_COUNT=${UWSGI_WORKERS_MAX} + - UWSGI_CHEAPER_WORKERS_COUNT=${UWSGI_WORKERS_START} + - UWSGI_MAX_REQUESTS=${UWSGI_MAX_REQUESTS} + - UWSGI_CHEAPER_RSS_LIMIT_SOFT=${UWSGI_SOFT_LIMIT} + - UWSGI_HARAKIRI=${UWSGI_HARAKIRI} + - UWSGI_WORKER_RELOAD_MERCY=${UWSGI_WORKER_RELOAD_MERCY} + - WSGI=${WSGI} + ${USE_CELERY} - SKIP_CELERY=True + ${USE_DEV_MODE} - DJANGO_SETTINGS_MODULE=kobo.settings.dev + ${USE_HTTPS} - SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO,https + ${USE_NPM_FROM_HOST} - FRONTEND_DEV_MODE=host ${USE_EXTRA_HOSTS}extra_hosts: ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} @@ -29,26 +30,18 @@ services: ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - kobocat - ${USE_BACKEND_NETWORK} - kobocat.docker.container + ${USE_BACKEND_NETWORK} - kpi + ${USE_BACKEND_NETWORK} - kpi.docker.container - kpi: + worker: ${USE_KPI_DEV_MODE} build: ${KPI_PATH} ${USE_KPI_DEV_MODE} image: kpi:dev.${KPI_DEV_BUILD_ID} ${USE_KPI_DEV_MODE} volumes: ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi environment: - - KPI_UWSGI_WORKERS_COUNT=${UWSGI_WORKERS_MAX} - - KPI_UWSGI_CHEAPER_WORKERS_COUNT=${UWSGI_WORKERS_START} - - NGINX_PUBLIC_PORT=${NGINX_PUBLIC_PORT} - - KPI_UWSGI_MAX_REQUESTS=${UWSGI_MAX_REQUESTS} - - KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT=${UWSGI_SOFT_LIMIT} - - KPI_UWSGI_HARAKIRI=${UWSGI_HARAKIRI} - - KPI_UWSGI_WORKER_RELOAD_MERCY=${UWSGI_WORKER_RELOAD_MERCY} - ${USE_CELERY} - SKIP_CELERY=True + - WSGI=${WSGI} ${USE_DEV_MODE} - DJANGO_SETTINGS_MODULE=kobo.settings.dev ${USE_HTTPS} - SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO,https - ${USE_NPM_FROM_HOST} - FRONTEND_DEV_MODE=host ${USE_EXTRA_HOSTS}extra_hosts: ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} @@ -60,13 +53,83 @@ services: ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} aliases: - ${USE_BACKEND_NETWORK} - kpi - ${USE_BACKEND_NETWORK} - kpi.docker.container + ${USE_BACKEND_NETWORK} - worker + ${USE_BACKEND_NETWORK} - worker.docker.container + + worker_kobocat: + ${USE_KPI_DEV_MODE} build: ${KPI_PATH} + ${USE_KPI_DEV_MODE} image: kpi:dev.${KPI_DEV_BUILD_ID} + ${USE_KPI_DEV_MODE} volumes: + ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi + environment: + - WSGI=${WSGI} + ${USE_DEV_MODE} - DJANGO_SETTINGS_MODULE=kobo.settings.dev + ${USE_HTTPS} - SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO,https + ${USE_EXTRA_HOSTS}extra_hosts: + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_BACKEND_NETWORK}networks: + ${USE_BACKEND_NETWORK} kobo-be-network: + ${USE_BACKEND_NETWORK} aliases: + ${USE_BACKEND_NETWORK} - worker_kobocat + ${USE_BACKEND_NETWORK} - worker_kobocat.docker.container + + worker_low_priority: + ${USE_KPI_DEV_MODE} build: ${KPI_PATH} + ${USE_KPI_DEV_MODE} image: kpi:dev.${KPI_DEV_BUILD_ID} + ${USE_KPI_DEV_MODE} volumes: + ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi + environment: + - WSGI=${WSGI} + ${USE_DEV_MODE} - DJANGO_SETTINGS_MODULE=kobo.settings.dev + ${USE_HTTPS} - SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO,https + ${USE_EXTRA_HOSTS}extra_hosts: + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_BACKEND_NETWORK}networks: + ${USE_BACKEND_NETWORK} kobo-be-network: + ${USE_BACKEND_NETWORK} aliases: + ${USE_BACKEND_NETWORK} - worker_low_priority + ${USE_BACKEND_NETWORK} - worker.docker.container + + beat: + ${USE_KPI_DEV_MODE} build: ${KPI_PATH} + ${USE_KPI_DEV_MODE} image: kpi:dev.${KPI_DEV_BUILD_ID} + ${USE_KPI_DEV_MODE} volumes: + ${USE_KPI_DEV_MODE} - ${KPI_PATH}:/srv/src/kpi + environment: + - WSGI=${WSGI} + ${USE_DEV_MODE} - DJANGO_SETTINGS_MODULE=kobo.settings.dev + ${USE_HTTPS} - SECURE_PROXY_SSL_HEADER=HTTP_X_FORWARDED_PROTO,https + ${USE_EXTRA_HOSTS}extra_hosts: + ${USE_FAKE_DNS} - ${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${USE_FAKE_DNS} - ${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}:${LOCAL_INTERFACE_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - postgres.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - mongo.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-main.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${ADD_BACKEND_EXTRA_HOSTS} - redis-cache.${PRIVATE_DOMAIN_NAME}:${PRIMARY_BACKEND_IP} + ${USE_BACKEND_NETWORK}networks: + ${USE_BACKEND_NETWORK} kobo-be-network: + ${USE_BACKEND_NETWORK} aliases: + ${USE_BACKEND_NETWORK} - beat + ${USE_BACKEND_NETWORK} - beat.docker.container nginx: environment: - NGINX_PUBLIC_PORT=${NGINX_PUBLIC_PORT} - UWSGI_PASS_TIMEOUT=${UWSGI_PASS_TIMEOUT} + - WSGI=${WSGI} ${USE_LETSENSCRYPT}ports: ${USE_LETSENSCRYPT} - ${NGINX_EXPOSED_PORT}:80 ${USE_EXTRA_HOSTS}extra_hosts: @@ -106,3 +169,4 @@ services: ${USE_BACKEND_NETWORK}networks: ${USE_BACKEND_NETWORK} kobo-be-network: ${USE_BACKEND_NETWORK} name: ${DOCKER_NETWORK_BACKEND_PREFIX}_kobo-be-network +${USE_BACKEND_NETWORK} external: true diff --git a/templates/kobo-env/envfile.txt.tpl b/templates/kobo-env/envfile.txt.tpl deleted file mode 100644 index 08242c90..00000000 --- a/templates/kobo-env/envfile.txt.tpl +++ /dev/null @@ -1,33 +0,0 @@ -####################### -# Mandatory variables # -####################### - -# Choose between http or https -PUBLIC_REQUEST_SCHEME=${PROTOCOL} -# The publicly-accessible domain where your KoBo Toolbox instance will be reached (e.g. example.com). -PUBLIC_DOMAIN_NAME=${PUBLIC_DOMAIN_NAME} -# The private domain used in docker network. Useful for communication between containers without passing through -# a load balancer. No need to be resolved by a public DNS. -INTERNAL_DOMAIN_NAME=${INTERNAL_DOMAIN_NAME} -# The publicly-accessible subdomain for the KoBoForm form building and management interface (e.g. koboform). -KOBOFORM_PUBLIC_SUBDOMAIN=${KOBOFORM_SUBDOMAIN} -# The publicly-accessible subdomain for the KoBoCAT data collection and project management interface (e.g.kobocat). -KOBOCAT_PUBLIC_SUBDOMAIN=${KOBOCAT_SUBDOMAIN} -# The publicly-accessible subdomain for the Enketo Express web forms (e.g. enketo). -ENKETO_EXPRESS_PUBLIC_SUBDOMAIN=${ENKETO_SUBDOMAIN} -# See "api key" here: https://github.com/kobotoolbox/enketo-express/tree/master/config#linked-form-and-data-server. -ENKETO_API_KEY=${ENKETO_API_KEY} -# Keep `ENKETO_API_TOKEN` for retro-compatibility with KPI and KoBoCAT. ToDo Remove when KPI and KC read correct env variable -ENKETO_API_TOKEN=${ENKETO_API_KEY} -# See "https://github.com/enketo/enketo-express/blob/master/setup/docker/create_config.py#L14 -ENKETO_ENCRYPTION_KEY=${ENKETO_ENCRYPTION_KEY} -# Canonically a 50-character random string. For Django 1.8.13, see https://docs.djangoproject.com/en/1.8/ref/settings/#secret-key and https://github.com/django/django/blob/4022b2c306e88a4ab7f80507e736ce7ac7d01186/django/core/management/commands/startproject.py#L29-L31. -# To generate a secret key in the same way as `django-admin startproject` you can run: -# docker-compose run --rm kpi python -c 'from django.utils.crypto import get_random_string; print(get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789!@#$$%^&*(-_=+)"))' -DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} -# The initial superuser's username. -KOBO_SUPERUSER_USERNAME=${KOBO_SUPERUSER_USERNAME} -# The initial superuser's password. -KOBO_SUPERUSER_PASSWORD=${KOBO_SUPERUSER_PASSWORD} -# The e-mail address where your users can contact you. -KOBO_SUPPORT_EMAIL=${DEFAULT_FROM_EMAIL} diff --git a/templates/kobo-env/envfiles/databases.txt.tpl b/templates/kobo-env/envfiles/databases.txt.tpl index bfa0951d..6e02336e 100644 --- a/templates/kobo-env/envfiles/databases.txt.tpl +++ b/templates/kobo-env/envfiles/databases.txt.tpl @@ -35,18 +35,11 @@ KPI_POSTGRES_DB=${KPI_POSTGRES_DB} # Postgres database used by kpi and kobocat Django apps KC_DATABASE_URL=postgis://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.${PRIVATE_DOMAIN_NAME}:${POSTGRES_PORT}/${KC_POSTGRES_DB} KPI_DATABASE_URL=postgis://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.${PRIVATE_DOMAIN_NAME}:${POSTGRES_PORT}/${KPI_POSTGRES_DB} - -# Replication -KOBO_POSTGRES_REPLICATION_USER=kobo_replication -KOBO_POSTGRES_REPLICATION_PASSWORD=${POSTGRES_REPLICATION_PASSWORD} -KOBO_POSTGRES_PRIMARY_ENDPOINT=primary.postgres.${PRIVATE_DOMAIN_NAME} +DATABASE_URL=postgis://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.${PRIVATE_DOMAIN_NAME}:${POSTGRES_PORT}/${KPI_POSTGRES_DB} # Default Postgres backup schedule is weekly at 02:00 AM UTC on Sunday. ${USE_BACKUP}POSTGRES_BACKUP_SCHEDULE=${POSTGRES_BACKUP_SCHEDULE} -# WAL-E archiving and backup support -${USE_WAL_E}USE_WAL_E=1 - #-------------------------------------------------------------------------------- # REDIS #-------------------------------------------------------------------------------- @@ -55,7 +48,6 @@ ${USE_WAL_E}USE_WAL_E=1 ${USE_BACKUP}REDIS_BACKUP_SCHEDULE=${REDIS_BACKUP_SCHEDULE} REDIS_SESSION_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/2 -REDIS_LOCK_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/3 REDIS_PASSWORD=${REDIS_PASSWORD} CACHE_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-cache.${PRIVATE_DOMAIN_NAME}:${REDIS_CACHE_PORT}/5 REDIS_CACHE_MAX_MEMORY=${REDIS_CACHE_MAX_MEMORY} diff --git a/templates/kobo-env/envfiles/django.txt.tpl b/templates/kobo-env/envfiles/django.txt.tpl new file mode 100644 index 00000000..cdd41127 --- /dev/null +++ b/templates/kobo-env/envfiles/django.txt.tpl @@ -0,0 +1,34 @@ +DJANGO_DEBUG=${DEBUG} +TEMPLATE_DEBUG=${DEBUG} +${USE_X_FORWARDED_HOST}USE_X_FORWARDED_HOST=True + +DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} +DJANGO_SESSION_COOKIE_AGE=${DJANGO_SESSION_COOKIE_AGE} +DJANGO_ALLOWED_HOSTS=.${PUBLIC_DOMAIN_NAME} .${INTERNAL_DOMAIN_NAME} +KPI_PREFIX=/ + +SESSION_COOKIE_DOMAIN=".${PUBLIC_DOMAIN_NAME}" + +CELERY_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/1 +CELERY_AUTOSCALE_MIN=2 +CELERY_AUTOSCALE_MAX=6 + +# Comma separated domains +${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOFORM_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME},${KOBOCAT_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} + +# See "api key" here: https://github.com/kobotoolbox/enketo-express/tree/master/config#linked-form-and-data-server. +ENKETO_API_KEY=${ENKETO_API_KEY} + +# The initial superuser's username. +KOBO_SUPERUSER_USERNAME=${KOBO_SUPERUSER_USERNAME} +# The initial superuser's password. +KOBO_SUPERUSER_PASSWORD=${KOBO_SUPERUSER_PASSWORD} +# The e-mail address where your users can contact you. +KOBO_SUPPORT_EMAIL=${DEFAULT_FROM_EMAIL} + +KOBOFORM_URL=${PUBLIC_REQUEST_SCHEME}://${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${NGINX_PUBLIC_PORT} +KOBOFORM_INTERNAL_URL=http://${KOBOFORM_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} # Always use HTTP internally. +ENKETO_URL=${PUBLIC_REQUEST_SCHEME}://${ENKETO_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${NGINX_PUBLIC_PORT} +ENKETO_INTERNAL_URL=http://${ENKETO_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} # Always use HTTP internally. +KOBOCAT_URL=${PUBLIC_REQUEST_SCHEME}://${KOBOCAT_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME}${NGINX_PUBLIC_PORT} +KOBOCAT_INTERNAL_URL=http://${KOBOCAT_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} # Always use HTTP internally. diff --git a/templates/kobo-env/envfiles/domains.txt.tpl b/templates/kobo-env/envfiles/domains.txt.tpl new file mode 100644 index 00000000..f41e6a5c --- /dev/null +++ b/templates/kobo-env/envfiles/domains.txt.tpl @@ -0,0 +1,13 @@ +# Choose between http or https +PUBLIC_REQUEST_SCHEME=${PUBLIC_REQUEST_SCHEME} +# The publicly-accessible domain where your KoBo Toolbox instance will be reached (e.g. example.com). +PUBLIC_DOMAIN_NAME=${PUBLIC_DOMAIN_NAME} +# The private domain used in docker network. Useful for communication between containers without passing through +# a load balancer. No need to be resolved by a public DNS. +INTERNAL_DOMAIN_NAME=${INTERNAL_DOMAIN_NAME} +# The publicly-accessible subdomain for the KoBoForm form building and management interface (e.g. koboform). +KOBOFORM_PUBLIC_SUBDOMAIN=${KOBOFORM_SUBDOMAIN} +# The publicly-accessible subdomain for the KoBoCAT data collection and project management interface (e.g.kobocat). +KOBOCAT_PUBLIC_SUBDOMAIN=${KOBOCAT_SUBDOMAIN} +# The publicly-accessible subdomain for the Enketo Express web forms (e.g. enketo). +ENKETO_EXPRESS_PUBLIC_SUBDOMAIN=${ENKETO_SUBDOMAIN} diff --git a/templates/kobo-env/envfiles/enketo.txt.tpl b/templates/kobo-env/envfiles/enketo.txt.tpl deleted file mode 100644 index 7e565924..00000000 --- a/templates/kobo-env/envfiles/enketo.txt.tpl +++ /dev/null @@ -1,5 +0,0 @@ -ENKETO_REDIS_MAIN_HOST=redis-main.${PRIVATE_DOMAIN_NAME} -ENKETO_REDIS_CACHE_HOST=redis-cache.${PRIVATE_DOMAIN_NAME} -ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL=${KOBOFORM_SUBDOMAIN}.${PUBLIC_DOMAIN_NAME} -ENKETO_LINKED_FORM_AND_DATA_SERVER_API_KEY=${ENKETO_API_KEY} -ENKETO_SUPPORT_EMAIL=${DEFAULT_FROM_EMAIL} diff --git a/templates/kobo-env/envfiles/external_services.txt.tpl b/templates/kobo-env/envfiles/external_services.txt.tpl index f83f1e6b..8fea3bb8 100644 --- a/templates/kobo-env/envfiles/external_services.txt.tpl +++ b/templates/kobo-env/envfiles/external_services.txt.tpl @@ -3,6 +3,5 @@ ############################################################################ GOOGLE_ANALYTICS_TOKEN=${GOOGLE_UA} -KOBOCAT_RAVEN_DSN=${KOBOCAT_RAVEN_DSN} -KPI_RAVEN_DSN=${KPI_RAVEN_DSN} -KPI_RAVEN_JS_DSN=${KPI_RAVEN_JS_DSN} \ No newline at end of file +SENTRY_DSN=${KPI_RAVEN_DSN} +SENTRY_JS_DSN=${KPI_RAVEN_JS_DSN} diff --git a/templates/kobo-env/envfiles/kobocat.txt.tpl b/templates/kobo-env/envfiles/kobocat.txt.tpl deleted file mode 100644 index b924290c..00000000 --- a/templates/kobo-env/envfiles/kobocat.txt.tpl +++ /dev/null @@ -1,20 +0,0 @@ -KOBOCAT_DJANGO_DEBUG=${DEBUG} -TEMPLATE_DEBUG=${DEBUG} -${USE_X_FORWARDED_HOST}USE_X_FORWARDED_HOST=True - -DJANGO_SESSION_COOKIE_AGE=${DJANGO_SESSION_COOKIE_AGE} -DJANGO_SETTINGS_MODULE=onadata.settings.prod -ENKETO_VERSION=Express - -KOBOCAT_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/2 -KOBOCAT_CELERY_LOG_FILE=/srv/logs/celery.log - -# Default KoBoCAT media backup schedule is weekly at 12:00 AM UTC on Sunday. -${USE_MEDIA_BACKUP}KOBOCAT_MEDIA_BACKUP_SCHEDULE=${KOBOCAT_MEDIA_BACKUP_SCHEDULE} - -# Dev: One or more mappings from PyDev remote debugging machine file paths to `kobocat` container -# file paths (see https://github.com/kobotoolbox/kobocat/blob/master/docker/setup_pydev.bash). -#KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS=~/devel/kobocat -> /srv/src/kobocat | ~/.virtualenvs/kobocat/lib/python2.7/site-packages -> /usr/local/lib/python2.7/dist-packages - -# Comma separated domains -${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOCAT_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} diff --git a/templates/kobo-env/envfiles/kpi.txt.tpl b/templates/kobo-env/envfiles/kpi.txt.tpl deleted file mode 100644 index d592c353..00000000 --- a/templates/kobo-env/envfiles/kpi.txt.tpl +++ /dev/null @@ -1,11 +0,0 @@ -KPI_DJANGO_DEBUG=${DEBUG} -TEMPLATE_DEBUG=${DEBUG} -${USE_X_FORWARDED_HOST}USE_X_FORWARDED_HOST=True - -DJANGO_SESSION_COOKIE_AGE=${DJANGO_SESSION_COOKIE_AGE} -ENKETO_VERSION=Express -KPI_PREFIX=/ -KPI_BROKER_URL=redis://{% if REDIS_PASSWORD %}:${REDIS_PASSWORD}@{% endif REDIS_PASSWORD %}redis-main.${PRIVATE_DOMAIN_NAME}:${REDIS_MAIN_PORT}/1 - -# Comma separated domains -${USE_SERVICE_ACCOUNT_WHITELISTED_HOSTS}SERVICE_ACCOUNT_WHITELISTED_HOSTS=${KOBOFORM_SUBDOMAIN}.${INTERNAL_DOMAIN_NAME} diff --git a/templates/kobo-env/envfiles/nginx.txt.tpl b/templates/kobo-env/envfiles/nginx.txt.tpl deleted file mode 100644 index fb164376..00000000 --- a/templates/kobo-env/envfiles/nginx.txt.tpl +++ /dev/null @@ -1,11 +0,0 @@ -# Options for the following are "uWSGI" or "runserver_plus" (for debugging). -KPI_WEB_SERVER=${WSGI_SERVER} -# django extensions are not installed on KoBoCat. So only `uWSGI` option is available. -KOBOCAT_WEB_SERVER=${WSGI_SERVER} - -# Options for the following are "Nginx" or "Django". -# NOTE: In order to serve static files from Django, the corresponding -# `..._DJANGO_DEBUG` environment variable must be set to "True" -# in `envfiles/kobocat.txt` and/or `envfiles/kpi.txt`. -KOBOCAT_STATIC_FILES_SERVER=Nginx -KPI_STATIC_FILES_SERVER=Nginx diff --git a/templates/kobo-env/postgres/primary/postgres.conf.tpl b/templates/kobo-env/postgres/conf/postgres.conf.tpl similarity index 52% rename from templates/kobo-env/postgres/primary/postgres.conf.tpl rename to templates/kobo-env/postgres/conf/postgres.conf.tpl index bd23009f..19617a38 100644 --- a/templates/kobo-env/postgres/primary/postgres.conf.tpl +++ b/templates/kobo-env/postgres/conf/postgres.conf.tpl @@ -1,7 +1,3 @@ -##################################################################################### -# PRIMARY SPECIFIC -# If file must be appended to shared/postgres.conf -##################################################################################### #------------------------------------------------------------------------------------ # TUNING #------------------------------------------------------------------------------------ @@ -14,7 +10,3 @@ # Total Memory (RAM): ${POSTGRES_RAM}GB ${POSTGRES_SETTINGS} - -${USE_WAL_E}archive_mode = on -${USE_WAL_E}archive_command = 'envdir $$PGDATA/wal-e.d/env wal-e wal-push %p' -${USE_WAL_E}archive_timeout = 60 diff --git a/templates/kobo-env/postgres/secondary/postgres.conf.tpl b/templates/kobo-env/postgres/secondary/postgres.conf.tpl deleted file mode 100644 index e2c84938..00000000 --- a/templates/kobo-env/postgres/secondary/postgres.conf.tpl +++ /dev/null @@ -1,26 +0,0 @@ -##################################################################################### -# SECONDARY SPECIFIC -# If file must be appended to shared/postgres.conf -##################################################################################### -#------------------------------------------------------------------------------------ -# TUNING -#------------------------------------------------------------------------------------ -# These settings are based on server configuration -# https://www.pgconfig.org/#/tuning -# DB Version: 14 -# OS Type: linux -# App profile: ${POSTGRES_APP_PROFILE} -# Hard-drive: SSD -# Total Memory (RAM): ${POSTGRES_RAM}GB - -${POSTGRES_SETTINGS} - -#------------------------------------------------------------------------------------ -# REPLICATION -#------------------------------------------------------------------------------------ -hot_standby_feedback = on - -# https://stackoverflow.com/a/33282856 -# https://stackoverflow.com/a/34404303 -max_standby_streaming_delay = -1 -max_standby_archive_delay = -1 diff --git a/templates/nginx-certbot/docker-compose.yml.tpl b/templates/nginx-certbot/docker-compose.yml.tpl index c7ff038a..49ddee2c 100644 --- a/templates/nginx-certbot/docker-compose.yml.tpl +++ b/templates/nginx-certbot/docker-compose.yml.tpl @@ -2,7 +2,7 @@ version: '3' services: nginx_ssl_proxy: - image: nginx:1.21-alpine + image: nginx:1.26-alpine restart: unless-stopped volumes: - ./data/nginx:/etc/nginx/conf.d diff --git a/tests/test_config.py b/tests/test_config.py index 8c88ac3f..e7dc38ae 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -57,61 +57,52 @@ def test_installation(): MagicMock(return_value=True)) def test_staging_mode(): config = read_config() - kc_repo_path = tempfile.mkdtemp() kpi_repo_path = tempfile.mkdtemp() with patch('helpers.cli.CLI.colored_input') as mock_colored_input: - mock_colored_input.side_effect = iter([CHOICE_YES, - kc_repo_path, - kpi_repo_path]) + mock_colored_input.side_effect = iter([CHOICE_YES, kpi_repo_path]) config._Config__questions_dev_mode() dict_ = config.get_dict() assert not config.dev_mode assert config.staging_mode - assert dict_['kpi_path'] == kpi_repo_path and \ - dict_['kc_path'] == kc_repo_path - - shutil.rmtree(kc_repo_path) + assert dict_['kpi_path'] == kpi_repo_path shutil.rmtree(kpi_repo_path) -@patch('helpers.config.Config._Config__clone_repo', - MagicMock(return_value=True)) +@patch('helpers.config.Config._Config__clone_repo', MagicMock(return_value=True)) def test_dev_mode(): config = test_installation() - kc_repo_path = tempfile.mkdtemp() kpi_repo_path = tempfile.mkdtemp() with patch('helpers.cli.CLI.colored_input') as mock_colored_input: - mock_colored_input.side_effect = iter(['8080', - CHOICE_YES, - CHOICE_NO, - kc_repo_path, - kpi_repo_path, - CHOICE_YES, - CHOICE_NO, - ]) + mock_colored_input.side_effect = iter( + [ + '8080', + CHOICE_YES, + CHOICE_NO, + kpi_repo_path, + CHOICE_YES, + CHOICE_NO, + ] + ) config._Config__questions_dev_mode() dict_ = config.get_dict() assert config.dev_mode assert not config.staging_mode assert config.get_dict().get('exposed_nginx_docker_port') == '8080' - assert dict_['kpi_path'] == kpi_repo_path and \ - dict_['kc_path'] == kc_repo_path + assert dict_['kpi_path'] == kpi_repo_path assert dict_['npm_container'] is False assert dict_['use_celery'] is False - shutil.rmtree(kc_repo_path) shutil.rmtree(kpi_repo_path) - with patch.object(CLI, 'colored_input', - return_value=CHOICE_NO) as mock_ci: + with patch.object(CLI, 'colored_input', return_value=CHOICE_NO) as mock_ci: config._Config__questions_dev_mode() dict_ = config.get_dict() assert not config.dev_mode - assert dict_['kpi_path'] == '' and dict_['kc_path'] == '' + assert dict_['kpi_path'] == '' def test_server_roles_questions(): @@ -121,7 +112,7 @@ def test_server_roles_questions(): with patch('helpers.cli.CLI.colored_input') as mock_colored_input: mock_colored_input.side_effect = iter( - [CHOICE_YES, 'frontend', 'backend', 'secondary']) + [CHOICE_YES, 'frontend', 'backend']) config._Config__questions_multi_servers() @@ -132,7 +123,6 @@ def test_server_roles_questions(): config._Config__questions_roles() assert not config.frontend assert config.backend - assert config.secondary_backend def test_session_cookies(): @@ -737,7 +727,7 @@ def test_update_postgres_username(): def test_update_postgres_db_name_from_single_database(): """ Simulate upgrade from single database to two databases. - With two databases, KoBoCat has its own database. We ensure that + With two databases, KoboCat has its own database. We ensure that `kc_postgres_db` gets `postgres_db` value. """ config = read_config() @@ -751,16 +741,6 @@ def test_update_postgres_db_name_from_single_database(): assert dict_['kc_postgres_db'] == old_db_name -def test_new_terminology(): - """ - Ensure config uses `primary` instead of `master` - """ - config = read_config() - config._Config__dict['backend_server_role'] = 'master' - dict_ = config.get_upgraded_dict() - assert dict_['backend_server_role'] == 'primary' - - def test_use_boolean(): """ Ensure config uses booleans instead of '1' or '2' @@ -811,8 +791,6 @@ def test_backup_schedules_from_single_instance(): # Force advanced options and single instance config._Config__dict['advanced'] = True config._Config__dict['multi'] = False - # Force `False` to validate it becomes `True` at the end - config._Config__dict['backup_from_primary'] = False assert config._Config__dict['kobocat_media_backup_schedule'] == '0 0 * * 0' assert config._Config__dict['mongo_backup_schedule'] == '0 1 * * 0' @@ -832,7 +810,6 @@ def test_backup_schedules_from_single_instance(): assert config._Config__dict['postgres_backup_schedule'] == '2 2 2 2 2' assert config._Config__dict['mongo_backup_schedule'] == '3 3 3 3 3' assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' - assert config._Config__dict['backup_from_primary'] is True def test_backup_schedules_from_frontend_instance(): @@ -860,7 +837,7 @@ def test_backup_schedules_from_frontend_instance(): assert config._Config__dict['kobocat_media_backup_schedule'] == '1 1 1 1 1' -def test_backup_schedules_from_primary_backend_with_redis_and_no_postgres(): +def test_backup_schedules_from_backend(): config = read_config() # Force advanced options config._Config__dict['advanced'] = True @@ -868,126 +845,26 @@ def test_backup_schedules_from_primary_backend_with_redis_and_no_postgres(): assert config._Config__dict['mongo_backup_schedule'] == '0 1 * * 0' assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' - assert config._Config__dict['run_redis_containers'] is True - assert config._Config__dict['backup_from_primary'] is True with patch('helpers.cli.CLI.colored_input') as mock_colored_input: - mock_colored_input.side_effect = iter( - [CHOICE_YES, 'backend', 'primary']) + mock_colored_input.side_effect = iter([CHOICE_YES, 'backend']) config._Config__questions_multi_servers() config._Config__questions_roles() - assert config.backend and config.primary_backend + assert config.backend with patch('helpers.cli.CLI.colored_input') as mock_ci: mock_ci.side_effect = iter([ CHOICE_YES, # Activate backup CHOICE_NO, # Choose AWS - CHOICE_NO, # Run postgres backup from current server + '1 1 1 1 1', # PostgreSQL '3 3 3 3 3', # Mongo '4 4 4 4 4', # Redis ]) config._Config__questions_backup() - assert config._Config__dict['postgres_backup_schedule'] == '' + assert config._Config__dict['postgres_backup_schedule'] == '1 1 1 1 1' assert config._Config__dict['mongo_backup_schedule'] == '3 3 3 3 3' assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' - assert config._Config__dict['backup_from_primary'] is False - - -def test_backup_schedules_from_primary_backend_with_no_redis_and_no_postgres(): - config = read_config() - # Force advanced options - config._Config__dict['advanced'] = True - config._Config__dict['run_redis_containers'] = False - - assert config._Config__dict['mongo_backup_schedule'] == '0 1 * * 0' - assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' - assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' - assert config._Config__dict['backup_from_primary'] is True - - with patch('helpers.cli.CLI.colored_input') as mock_colored_input: - mock_colored_input.side_effect = iter( - [CHOICE_YES, 'backend', 'primary']) - config._Config__questions_multi_servers() - config._Config__questions_roles() - assert config.backend and config.primary_backend - - with patch('helpers.cli.CLI.colored_input') as mock_ci: - mock_ci.side_effect = iter([ - CHOICE_YES, # Activate backup - CHOICE_NO, # Choose AWS - CHOICE_NO, # Run postgres backup from current server - '3 3 3 3 3', # Mongo - ]) - config._Config__questions_backup() - - assert config._Config__dict['postgres_backup_schedule'] == '' - assert config._Config__dict['mongo_backup_schedule'] == '3 3 3 3 3' - assert config._Config__dict['redis_backup_schedule'] == '' - assert config._Config__dict['backup_from_primary'] is False - - -def test_backup_schedules_from_secondary_backend_with_redis_and_postgres(): - config = read_config() - # Force advanced options - config._Config__dict['advanced'] = True - config._Config__dict['run_redis_containers'] = True - - assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' - assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' - assert config._Config__dict['backup_from_primary'] is True - - with patch('helpers.cli.CLI.colored_input') as mock_colored_input: - mock_colored_input.side_effect = iter( - [CHOICE_YES, 'backend', 'secondary']) - config._Config__questions_multi_servers() - config._Config__questions_roles() - assert config.backend and config.secondary_backend - - with patch('helpers.cli.CLI.colored_input') as mock_ci: - mock_ci.side_effect = iter([ - CHOICE_YES, # Activate backup - CHOICE_NO, # Choose AWS - CHOICE_YES, # Run postgres backup from current server - '2 2 2 2 2', # PostgreSQL - '4 4 4 4 4', # Redis - ]) - config._Config__questions_backup() - - assert config._Config__dict['postgres_backup_schedule'] == '2 2 2 2 2' - assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' - assert config._Config__dict['backup_from_primary'] is False - - -def test_backup_schedules_from_secondary_backend_with_redis_and_no_postgres(): - config = read_config() - # Force advanced options - config._Config__dict['advanced'] = True - config._Config__dict['run_redis_containers'] = True - - assert config._Config__dict['postgres_backup_schedule'] == '0 2 * * 0' - assert config._Config__dict['redis_backup_schedule'] == '0 3 * * 0' - - with patch('helpers.cli.CLI.colored_input') as mock_colored_input: - mock_colored_input.side_effect = iter( - [CHOICE_YES, 'backend', 'secondary']) - config._Config__questions_multi_servers() - config._Config__questions_roles() - assert config.backend and config.secondary_backend - - with patch('helpers.cli.CLI.colored_input') as mock_ci: - mock_ci.side_effect = iter([ - CHOICE_YES, # Activate backup - CHOICE_NO, # Choose AWS - CHOICE_NO, # Run postgres backup from current server - '4 4 4 4 4', # Redis - ]) - config._Config__questions_backup() - - assert config._Config__dict['postgres_backup_schedule'] == '' - assert config._Config__dict['redis_backup_schedule'] == '4 4 4 4 4' - assert config._Config__dict['backup_from_primary'] is True - assert config._Config__dict['run_redis_containers'] is True def test_activate_only_postgres_backup(): diff --git a/tests/test_run.py b/tests/test_run.py index 7120b115..deecb541 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -23,7 +23,7 @@ def test_toggle_trivial(): Command.start() mock_docker = MockDocker() expected_containers = MockDocker.FRONTEND_CONTAINERS + \ - MockDocker.PRIMARY_BACKEND_CONTAINERS + \ + MockDocker.BACKEND_CONTAINERS + \ MockDocker.LETSENCRYPT assert sorted(mock_docker.ps()) == sorted(expected_containers) @@ -45,8 +45,9 @@ def test_toggle_no_letsencrypt(): config_object._Config__dict['use_letsencrypt'] = False Command.start() mock_docker = MockDocker() - expected_containers = MockDocker.FRONTEND_CONTAINERS + \ - MockDocker.PRIMARY_BACKEND_CONTAINERS + expected_containers = ( + MockDocker.FRONTEND_CONTAINERS + MockDocker.BACKEND_CONTAINERS + ) assert sorted(mock_docker.ps()) == sorted(expected_containers) Command.stop() @@ -83,39 +84,14 @@ def test_toggle_frontend(): MagicMock(return_value=True)) @patch('helpers.cli.CLI.run_command', new=MockCommand.run_command) -def test_toggle_primary_backend(): +def test_toggle_backend(): config_object = read_config() - config_object._Config__dict['backend_server_role'] = 'primary' config_object._Config__dict['server_role'] = 'backend' config_object._Config__dict['multi'] = True Command.start() mock_docker = MockDocker() - expected_containers = MockDocker.PRIMARY_BACKEND_CONTAINERS - assert sorted(mock_docker.ps()) == sorted(expected_containers) - - Command.stop() - assert len(mock_docker.ps()) == 0 - del mock_docker - - -@patch('helpers.network.Network.is_port_open', - MagicMock(return_value=False)) -@patch('helpers.command.Upgrading.migrate_single_to_two_databases', - new=MockUpgrading.migrate_single_to_two_databases) -@patch('helpers.command.Command.info', - MagicMock(return_value=True)) -@patch('helpers.cli.CLI.run_command', - new=MockCommand.run_command) -def test_toggle_secondary_backend(): - config_object = read_config() - config_object._Config__dict['backend_server_role'] = 'secondary' - config_object._Config__dict['server_role'] = 'backend' - config_object._Config__dict['multi'] = True - - mock_docker = MockDocker() - Command.start() - expected_containers = MockDocker.SECONDARY_BACKEND_CONTAINERS + expected_containers = MockDocker.BACKEND_CONTAINERS assert sorted(mock_docker.ps()) == sorted(expected_containers) Command.stop() @@ -135,16 +111,20 @@ def test_toggle_maintenance(): config_object = read_config() mock_docker = MockDocker() Command.start() - expected_containers = MockDocker.FRONTEND_CONTAINERS + \ - MockDocker.PRIMARY_BACKEND_CONTAINERS + \ - MockDocker.LETSENCRYPT + expected_containers = ( + MockDocker.FRONTEND_CONTAINERS + + MockDocker.BACKEND_CONTAINERS + + MockDocker.LETSENCRYPT + ) assert sorted(mock_docker.ps()) == sorted(expected_containers) config_object._Config__dict['maintenance_enabled'] = True Command.start() - maintenance_containers = MockDocker.PRIMARY_BACKEND_CONTAINERS + \ - MockDocker.MAINTENANCE_CONTAINERS + \ - MockDocker.LETSENCRYPT + maintenance_containers = ( + MockDocker.BACKEND_CONTAINERS + + MockDocker.MAINTENANCE_CONTAINERS + + MockDocker.LETSENCRYPT + ) assert sorted(mock_docker.ps()) == sorted(maintenance_containers) config_object._Config__dict['maintenance_enabled'] = False Command.start() @@ -152,4 +132,3 @@ def test_toggle_maintenance(): Command.stop() assert len(mock_docker.ps()) == 0 del mock_docker - diff --git a/tests/utils.py b/tests/utils.py index 98daeafb..422b5083 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -72,11 +72,12 @@ def run_command(cls, command, cwd=None, polling=False): class MockDocker(metaclass=Singleton): - PRIMARY_BACKEND_CONTAINERS = ['primary_postgres', - 'mongo', - 'redis_main', - 'redis_cache'] - SECONDARY_BACKEND_CONTAINERS = ['secondary_postgres'] + BACKEND_CONTAINERS = [ + 'primary_postgres', + 'mongo', + 'redis_main', + 'redis_cache', + ] FRONTEND_CONTAINERS = ['nginx', 'kobocat', 'kpi', 'enketo_express'] MAINTENANCE_CONTAINERS = ['maintenance', 'kobocat', 'kpi', 'enketo_express'] LETSENCRYPT = ['letsencrypt_nginx', 'certbot'] @@ -98,10 +99,8 @@ def compose(self, command, cwd): if command[-2] == 'up': if letsencrypt: self.__containers += self.LETSENCRYPT - elif 'primary' in command[2]: - self.__containers += self.PRIMARY_BACKEND_CONTAINERS - elif 'secondary' in command[2]: - self.__containers += self.SECONDARY_BACKEND_CONTAINERS + elif 'backend' in command[2]: + self.__containers += self.BACKEND_CONTAINERS elif 'maintenance' in command[2]: self.__containers += self.MAINTENANCE_CONTAINERS elif 'frontend' in command[2]: @@ -111,11 +110,8 @@ def compose(self, command, cwd): if letsencrypt: for container in self.LETSENCRYPT: self.__containers.remove(container) - elif 'primary' in command[2]: - for container in self.PRIMARY_BACKEND_CONTAINERS: - self.__containers.remove(container) - elif 'secondary' in command[2]: - for container in self.SECONDARY_BACKEND_CONTAINERS: + elif 'backend' in command[2]: + for container in self.BACKEND_CONTAINERS: self.__containers.remove(container) elif 'maintenance' in command[2]: for container in self.MAINTENANCE_CONTAINERS: diff --git a/tox.ini b/tox.ini index f73f663f..490464b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ # content of: tox.ini , put in same dir as setup.py [tox] skipsdist=True -envlist = py36,py37,py38 +envlist = py38,py310,py312 [testenv] deps = -rrequirements_tests.txt