From 16ba919ccf16fe2808553b0df5922e9705ed9194 Mon Sep 17 00:00:00 2001 From: zsdc Date: Wed, 5 Jul 2023 16:49:09 +0300 Subject: [PATCH] VPP: T1797: Optimized memory allocation during startup - changed memory requirement from total to available. This allows to start on systems with less total memory and protects from startup on systems with overloaded memory. - prevent startup if sysctl settings were not applied. This protects from situations when the system cannot allocate enough hugepages or apply other sysctl settings. --- data/templates/vpp/sysctl.conf.j2 | 15 ---------- src/conf_mode/vpp.py | 48 +++++++++++++++++++------------ 2 files changed, 29 insertions(+), 34 deletions(-) delete mode 100644 data/templates/vpp/sysctl.conf.j2 diff --git a/data/templates/vpp/sysctl.conf.j2 b/data/templates/vpp/sysctl.conf.j2 deleted file mode 100644 index 2207e2e3824..00000000000 --- a/data/templates/vpp/sysctl.conf.j2 +++ /dev/null @@ -1,15 +0,0 @@ -# Number of 2MB hugepages desired -vm.nr_hugepages=1024 - -# Must be greater than or equal to (2 * vm.nr_hugepages). -vm.max_map_count=3096 - -# All groups allowed to access hugepages -vm.hugetlb_shm_group=0 - -# Shared Memory Max must be greater or equal to the total size of hugepages. -# For 2MB pages, TotalHugepageSize = vm.nr_hugepages * 2 * 1024 * 1024 -# If the existing kernel.shmmax setting (cat /proc/sys/kernel/shmmax) -# is greater than the calculated TotalHugepageSize then set this parameter -# to current shmmax value. -kernel.shmmax=2147483648 diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index dc13f4e60cb..3c87a24aebd 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import psutil +from psutil import virtual_memory from pathlib import Path from re import search as re_search, MULTILINE as re_M @@ -26,6 +26,7 @@ from vyos.configdict import node_changed from vyos.ifconfig import Section from vyos.util import call, rc_cmd, boot_configuration_complete +from vyos.utils.system import sysctl_read, sysctl_apply from vyos.template import render from vyos.xml import defaults @@ -39,10 +40,10 @@ service_name = 'vpp' service_conf = Path(f'/run/vpp/{service_name}.conf') systemd_override = '/run/systemd/system/vpp.service.d/10-override.conf' -sysctl_vpp = '/etc/sysctl.d/80-vpp.conf' -# Min memory 6GB (2GB reserved for vpp) -MIN_TOTAL_MEMORY = 6 +# Free memory required for VPP +# 2 GB for hugepages + 1 GB for other services +MIN_AVAILABLE_MEMORY: int = 3 * 1024**3 def _get_pci_address_by_interface(iface) -> str: @@ -64,7 +65,6 @@ def _get_pci_address_by_interface(iface) -> str: raise ConfigError(f'Cannot find PCI address for interface {iface}') - def get_config(config=None): if config: conf = config @@ -131,32 +131,44 @@ def verify(config): return None if 'interface' not in config: - raise ConfigError(f'"interface" is required but not set!') + raise ConfigError('"interface" is required but not set!') if 'cpu' in config: - if 'corelist_workers' in config['cpu'] and 'main_core' not in config['cpu']: - raise ConfigError(f'"cpu main-core" is required but not set!') + if 'corelist_workers' in config['cpu'] and 'main_core' not in config[ + 'cpu']: + raise ConfigError('"cpu main-core" is required but not set!') - memory = psutil.virtual_memory() - memory_total = round(memory.total / (1024 ** 3), 2) - if memory_total < MIN_TOTAL_MEMORY: + memory_available: int = virtual_memory().available + if memory_available < MIN_AVAILABLE_MEMORY: raise ConfigError( - f'Not enough installed memory {memory_total}GB! ' - f'The minimum required memory is {MIN_TOTAL_MEMORY}GB.' - ) + 'Not enough free memory to start VPP:\n' + f'available: {round(memory_available / 1024**3, 1)}GB\n' + f'required: {round(MIN_AVAILABLE_MEMORY / 1024**3, 1)}GB') def generate(config): if not config or (len(config) == 1 and 'removed_ifaces' in config): # Remove old config and return service_conf.unlink(missing_ok=True) - if os.path.isfile(sysctl_vpp): - os.unlink(sysctl_vpp) return None render(service_conf, 'vpp/startup.conf.j2', config) render(systemd_override, 'vpp/override.conf.j2', config) - render(sysctl_vpp, 'vpp/sysctl.conf.j2', config) + + # apply sysctl settings + sysctl_config: dict[str, str] = { + 'vm.nr_hugepages': '1024', + 'vm.max_map_count': '3096', + 'vm.hugetlb_shm_group': '0', + 'kernel.shmmax': '2147483648' + } + # we do not want to reduce `kernel.shmmax` + kernel_shmnax_current: str = sysctl_read('kernel.shmmax') + if int(kernel_shmnax_current) > int(sysctl_config['kernel.shmmax']): + sysctl_config['kernel.shmmax'] = kernel_shmnax_current + + if not sysctl_apply(sysctl_config): + raise ConfigError('Cannot configure sysctl parameters for VPP') return None @@ -168,8 +180,6 @@ def apply(config): call('systemctl daemon-reload') call(f'systemctl restart {service_name}.service') - call(f'sysctl -qp {sysctl_vpp}') - # Initialize interfaces removed from VPP for iface in config.get('removed_ifaces', []): host_control = HostControl()