Skip to content

Commit

Permalink
Merge pull request #214 from kobotoolbox/213-postgres-settings
Browse files Browse the repository at this point in the history 
Make PostgreSQL default settings consistent
  • Loading branch information
@jnm
jnm authored Feb 1, 2023
2 parents 43054c5 + 3f3cc7a commit b77b437
Showing 1 changed file with 30 additions and 70 deletions.
100 changes: 30 additions & 70 deletions helpers/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,14 +380,14 @@ def get_template(cls):
'postgres_replication_password': Config.generate_password(),
'postgres_settings': False,
'postgres_settings_content': '\n'.join([
'# Generated by PGConfig 3.0.0 (3af8ea764b2fd9ea8401acfdc69f90cb825cdee9)',
'# https://api.pgconfig.org/v1/tuning/get-config?environment_name=Desktop&format=conf&include_pgbadger=false&cpus=1&max_connections=100&pg_version=14&total_ram=2GB&drive_type=HDD',
'# Generated by PGConfig 3.1.0 (1d600ea0d1d79f13dd7ed686f9e2befc1fcf9226)',
'# https://api.pgconfig.org/v1/tuning/get-config?format=conf&include_pgbadger=false&max_connections=100&pg_version=14&environment_name=Mixed&total_ram=2GB&cpus=1&drive_type=SSD&os_type=linux',
'',
'# Memory Configuration',
'shared_buffers = 128MB',
'effective_cache_size = 307MB',
'work_mem = 419KB',
'maintenance_work_mem = 20MB',
'shared_buffers = 256MB',
'effective_cache_size = 768MB',
'work_mem = 2MB',
'maintenance_work_mem = 51MB',
'',
'# Checkpoint Related Configuration',
'min_wal_size = 2GB',
Expand All @@ -400,8 +400,8 @@ def get_template(cls):
'max_connections = 100',
'',
'# Storage Configuration',
'random_page_cost = 4.0',
'effective_io_concurrency = 2',
'random_page_cost = 1.1',
'effective_io_concurrency = 200',
'',
'# Worker Processes Configuration',
'max_worker_processes = 8',
Expand Down Expand Up @@ -1586,35 +1586,13 @@ def __questions_postgres(self):
'Do you want to tweak PostgreSQL settings?',
default=self.__dict['postgres_settings']
)

template = self.get_template()

if self.__dict['postgres_settings']:

CLI.colored_print('Launching pgconfig.org API container...',
CLI.COLOR_INFO)
# pgconfig.org API is often unresponsive and make kobo-install
# hang forever.
# A docker image is available, let's use it instead.
# (Hope docker hub is not down too).

# Find an open port.
open_port = 9080
while True:
if not Network.is_port_open(open_port):
break
open_port += 1

# Unfortunately, the docker image is old and does not support
# PostgreSQL 14. The author has rewritten the code base in Go
# (instead of Python), so chances are pretty low that the image
# gets updated someday.
# ToDo Keep the code below for few months just in case and
# get rid of it if it never happens.

# # Start pgconfig.org API docker image
# docker_command = ['docker', 'run', '--rm', '-p',
# f'127.0.0.1:{open_port}:8080',
# '-d', '--name', 'pgconfig_container',
# 'sebastianwebber/pgconfig-api']
# CLI.run_command(docker_command)

# From https://docs.pgconfig.org/api/#available-parameters
# Parameters are case-sensitive, for example
Expand Down Expand Up @@ -1650,19 +1628,24 @@ def __questions_postgres(self):
self.__dict['postgres_max_connections'])

if self.multi_servers:
multi_servers_profiles = ['web', 'oltp', 'dw']
if self.__dict['postgres_profile'].lower() \
not in multi_servers_profiles:
self.__dict['postgres_profile'] = 'web'
multi_servers_profiles = ['web', 'oltp', 'dw', 'mixed']
if (
self.__dict['postgres_profile'].lower()
not in multi_servers_profiles
):
self.__dict['postgres_profile'] = template[
'postgres_profile'
]

CLI.colored_print('Application profile?', CLI.COLOR_QUESTION)
CLI.colored_print('\tweb) General Web application')
CLI.colored_print(
'\toltp) ERP or long transaction applications')
CLI.colored_print('\tdw) DataWare house')
CLI.colored_print('\tmixed) DB and APP on the same server')

self.__dict['postgres_profile'] = CLI.get_response(
['web', 'oltp', 'dw'],
['web', 'oltp', 'dw', 'mixed'],
self.__dict['postgres_profile'].lower())

self.__dict['postgres_profile'] = self.__dict[
Expand All @@ -1673,8 +1656,7 @@ def __questions_postgres(self):
else:
self.__dict['postgres_profile'] = 'Mixed'

# Instead of using the API locally with the docker image, let's
# use the (unreliable) public API.
# Use pgconfig.org API to get the configuration
# Notes: It has failed several times in the past.
endpoint = (
'https://api.pgconfig.org/v1/tuning/get-config'
Expand All @@ -1685,9 +1667,9 @@ def __questions_postgres(self):
'&pg_version=14'
'&total_ram={ram}GB'
'&drive_type={drive_type}'
'&os_type=linux'
)
endpoint = endpoint.format(
open_port=open_port,
profile=self.__dict['postgres_profile'],
ram=self.__dict['postgres_ram'],
cpus=self.__dict['postgres_cpus'],
Expand All @@ -1696,44 +1678,22 @@ def __questions_postgres(self):
)
response = Network.curl(endpoint)
if response:
self.__dict['postgres_settings_content'] = re.sub(
r'(log|lc_).+(\n|$)', '', response)
# Patch response because of https://github.com/pgconfig/api/issues/13
configuration = re.sub(r'(\d+)KB', "\1kB", configuration)
self.__dict['postgres_settings_content'] = configuration
else:
CLI.colored_print('\nAn error has occurred. Current '
'PostgreSQL settings will be used',
CLI.COLOR_INFO)

# ToDo Keep the code below for few months just in case and
# get rid of it if it never happens.
# # Stop container
# docker_command = ['docker', 'stop', '-t', '0',
# 'pgconfig_container']
# CLI.run_command(docker_command)
# CLI.colored_print('pgconfig.org API container has been stopped!',
# CLI.COLOR_INFO)
else:
# Forcing the default settings to remain even if there
# is an existing value in .run.conf. Without this,
# the value for `postgres_settings_content` would not update
default_postgres_settings_content = '\n'.join([
'# Memory Configuration',
'shared_buffers = 512MB',
'effective_cache_size = 2GB',
'work_mem = 10MB',
'maintenance_work_mem = 128MB',
'',
'# Checkpoint Related Configuration',
'min_wal_size = 512MB',
'max_wal_size = 2GB',
'checkpoint_completion_target = 0.9',
'wal_buffers = 15MB',
'',
'# Network Related Configuration',
"listen_addresses = '*'",
'max_connections = 100',
])
self.__dict['postgres_settings_content'] = \
default_postgres_settings_content

self.__dict['postgres_settings_content'] = template[
'postgres_settings_content'
]

def __questions_ports(self):
"""
Expand Down

0 comments on commit b77b437

Please sign in to comment.