From b1912c75c745a47d73a14f26d3e05ebbcda63b70 Mon Sep 17 00:00:00 2001 From: Florian Paul Azim Hoberg Date: Fri, 12 Jul 2024 14:52:23 +0200 Subject: [PATCH] feature: Add configurable log verbosity and vm rebalancing by total usage Fixes #16 Fixes #17 --- .../1.0.0/16_rebalance_vms_by_total_value.yml | 2 + .../1.0.0/17_make_logging_configurable.yml | 4 ++ README.md | 16 ++---- proxlb | 51 ++++++++++++------- proxlb.conf | 2 + 5 files changed, 45 insertions(+), 30 deletions(-) create mode 100644 .changelogs/1.0.0/16_rebalance_vms_by_total_value.yml create mode 100644 .changelogs/1.0.0/17_make_logging_configurable.yml diff --git a/.changelogs/1.0.0/16_rebalance_vms_by_total_value.yml b/.changelogs/1.0.0/16_rebalance_vms_by_total_value.yml new file mode 100644 index 0000000..49acea1 --- /dev/null +++ b/.changelogs/1.0.0/16_rebalance_vms_by_total_value.yml @@ -0,0 +1,2 @@ +added: + - Add option to rebalance VMs by their total value instead of used. [#16] diff --git a/.changelogs/1.0.0/17_make_logging_configurable.yml b/.changelogs/1.0.0/17_make_logging_configurable.yml new file mode 100644 index 0000000..5d2db89 --- /dev/null +++ b/.changelogs/1.0.0/17_make_logging_configurable.yml @@ -0,0 +1,4 @@ +added: + - Add feature to make log verbosity configurable [#17]. +changed: + - Adjusted general logging and log more details. diff --git a/README.md b/README.md index a75672e..5b4e20a 100644 --- a/README.md +++ b/README.md @@ -80,32 +80,24 @@ The following options can be set in the `proxlb.conf` file: | api_pass | FooBar | Password for the API. | | verify_ssl | 1 | Validate SSL certificates (1) or ignore (0). (default: 1) | | method | memory | Defines the balancing method (default: memory) where you can use `memory`, `disk` or `cpu`. | +| mode | used | Defines the balancing mode (default: `used`) where you can use `used` or `total` | | balanciness | 10 | Value of the percentage of lowest and highest resource consumption on nodes may differ before rebalancing. (default: 10) | | ignore_nodes | dummynode01,dummynode02,test* | Defines a comma separated list of nodes to exclude. | | ignore_vms | testvm01,testvm02 | Defines a comma separated list of VMs to exclude. (`*` as suffix wildcard or tags are also supported) | | daemon | 1 | Run as a daemon (1) or one-shot (0). (default: 1) | | schedule | 24 | Hours to rebalance in hours. (default: 24) | +| log_verbosity | INFO | Defines the log level (default: CRITICAL) where you can use `INFO`, `WARN` or `CRITICAL` | -An example of the configuration file looks like: +A minimal example of the configuration file looks like: ``` [proxmox] api_host: hypervisor01.gyptazy.ch api_user: root@pam api_pass: FooBar -verify_ssl: 1 [balancing] method: memory -# Balanciness defines how much difference may be -# between the lowest & highest resource consumption -# of nodes before rebalancing will be done. -# Examples: -# Rebalancing: node01: 41% memory consumption :: node02: 52% consumption -# No rebalancing: node01: 43% memory consumption :: node02: 50% consumption -balanciness: 10 -ignore_nodes: dummynode01,dummynode02 -ignore_vms: testvm01,testvm02 [service] -daemon: 1 +daemon: 0 ``` ### Parameters diff --git a/proxlb b/proxlb index 57aa7a3..7a07f82 100755 --- a/proxlb +++ b/proxlb @@ -72,15 +72,18 @@ class SystemdHandler(logging.Handler): # Functions -def initialize_logger(log_level, log_handler): +def initialize_logger(log_level, update_log_verbosity=False): """ Initialize ProxLB logging handler. """ info_prefix = 'Info: [logger]:' root_logger = logging.getLogger() root_logger.setLevel(log_level) - root_logger.addHandler(SystemdHandler()) - logging.info(f'{info_prefix} Logger got initialized.') + if not update_log_verbosity: + root_logger.addHandler(SystemdHandler()) + logging.info(f'{info_prefix} Logger got initialized.') + else: + logging.info(f'{info_prefix} Logger verbosity got updated to: {log_level}.') def pre_validations(config_path): """ Run pre-validations as sanity checks. """ @@ -175,12 +178,14 @@ def initialize_config_options(config_path): proxmox_api_ssl_v = config['proxmox']['verify_ssl'] # Balancing balancing_method = config['balancing'].get('method', 'memory') + balancing_mode = config['balancing'].get('mode', 'used_resources') balanciness = config['balancing'].get('balanciness', 10) ignore_nodes = config['balancing'].get('ignore_nodes', None) ignore_vms = config['balancing'].get('ignore_vms', None) # Service daemon = config['service'].get('daemon', 1) schedule = config['service'].get('schedule', 24) + log_verbosity = config['service'].get('log_verbosity', 'CRITICAL') except configparser.NoSectionError: logging.critical(f'{error_prefix} Could not find the required section.') sys.exit(2) @@ -193,7 +198,7 @@ def initialize_config_options(config_path): logging.info(f'{info_prefix} Configuration file loaded.') return proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v, balancing_method, \ - balanciness, ignore_nodes, ignore_vms, daemon, schedule + balancing_mode, balanciness, ignore_nodes, ignore_vms, daemon, schedule, log_verbosity def api_connect(proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v): @@ -350,7 +355,7 @@ def __get_proxlb_groups(vm_tags): return group_include, group_exclude, vm_ignore -def balancing_calculations(balancing_method, node_statistics, vm_statistics, balanciness): +def balancing_calculations(balancing_method, node_statistics, vm_statistics, balanciness, balancing_mode ): """ Calculate re-balancing of VMs on present nodes across the cluster. """ info_prefix = 'Info: [rebalancing-calculator]:' balanciness = int(balanciness) @@ -359,6 +364,11 @@ def balancing_calculations(balancing_method, node_statistics, vm_statistics, bal rebalance = True emergency_counter = 0 + # Log balancing information + logging.info(f'{info_prefix} Rebalancing will be done for method: {balancing_method}.') + logging.info(f'{info_prefix} Rebalancing will be done by: {balancing_mode} resources.') + logging.info(f'{info_prefix} Balanciness is set to: {balanciness}.') + # Validate for a supported balancing method. __validate_balancing_method(balancing_method) @@ -369,7 +379,7 @@ def balancing_calculations(balancing_method, node_statistics, vm_statistics, bal rebalance = __validate_balanciness(balanciness, balancing_method, node_statistics) if rebalance: - resource_highest_used_resources_vm, processed_vms = __get_most_used_resources_vm(balancing_method, vm_statistics, processed_vms) + resource_highest_used_resources_vm, processed_vms = __get_most_used_resources_vm(balancing_method, vm_statistics, processed_vms, balancing_mode) resource_highest_free_resources_node = __get_most_free_resources_node(balancing_method, node_statistics) node_statistics, vm_statistics = __update_resource_statistics(resource_highest_used_resources_vm, resource_highest_free_resources_node, vm_statistics, node_statistics, balancing_method) @@ -401,7 +411,7 @@ def __validate_balancing_method(balancing_method): def __validate_balanciness(balanciness, balancing_method, node_statistics): """ Validate for balanciness to ensure further rebalancing is needed. """ - info_prefix = 'Info: [balanciness-validation]]:' + info_prefix = 'Info: [balanciness-validation]:' node_memory_free_percent_list = [] for node_name, node_info in node_statistics.items(): @@ -412,25 +422,28 @@ def __validate_balanciness(balanciness, balancing_method, node_statistics): node_highest_percent = node_memory_free_percent_list_sorted[-1] if (node_lowest_percent + balanciness) < node_highest_percent: - logging.info(f'{info_prefix} Rebalancing is for {balancing_method} is needed.') + logging.info(f'{info_prefix} Rebalancing is for {balancing_method} is needed. Highest usage: {node_lowest_percent}% | Lowest usage: {node_lowest_percent}') return True else: - logging.info(f'{info_prefix} Rebalancing is for {balancing_method} is not needed.') + logging.info(f'{info_prefix} Rebalancing is for {balancing_method} is not needed. Highest usage: {node_lowest_percent}% | Lowest usage: {node_lowest_percent}') return False -def __get_most_used_resources_vm(balancing_method, vm_statistics, processed_vms): +def __get_most_used_resources_vm(balancing_method, vm_statistics, processed_vms, balancing_mode): """ Get and return the most used resources of a VM by the defined balancing method. """ + info_prefix = 'Info: [get-used-resources-vm]:' + if balancing_method == 'memory': - vm = max(vm_statistics.items(), key=lambda item: item[1]['memory_used'] if item[0] not in processed_vms else -float('inf')) + vm = max(vm_statistics.items(), key=lambda item: item[1][f'memory_{balancing_mode}'] if item[0] not in processed_vms else -float('inf')) + logging.info(f'{info_prefix} {vm}.') processed_vms.append(vm[0]) return vm, processed_vms if balancing_method == 'disk': - vm = max(vm_statistics.items(), key=lambda item: item[1]['disk_used'] if item[0] not in processed_vms else -float('inf')) + vm = max(vm_statistics.items(), key=lambda item: item[1][f'disk_{balancing_mode}'] if item[0] not in processed_vms else -float('inf')) processed_vms.append(vm[0]) return vm, processed_vms if balancing_method == 'cpu': - vm = max(vm_statistics.items(), key=lambda item: item[1]['cpu_used'] if item[0] not in processed_vms else -float('inf')) + vm = max(vm_statistics.items(), key=lambda item: item[1][f'cpu_{balancing_mode}'] if item[0] not in processed_vms else -float('inf')) processed_vms.append(vm[0]) return vm, processed_vms @@ -601,15 +614,17 @@ def print_table_cli(table): def main(): """ Run ProxLB for balancing VM workloads across a Proxmox cluster. """ - # Initialize PAS. - initialize_logger('CRITICAL', 'SystemdHandler()') app_args = initialize_args() + initialize_logger('CRITICAL') config_path = initialize_config_path(app_args) pre_validations(config_path) # Parse global config proxmox_api_host, proxmox_api_user, proxmox_api_pass, proxmox_api_ssl_v, balancing_method, \ - balanciness, ignore_nodes, ignore_vms, daemon, schedule = initialize_config_options(config_path) + balancing_mode, balanciness, ignore_nodes, ignore_vms, daemon, schedule, log_verbosity = initialize_config_options(config_path) + + # Overwrite logging handler with user defined log verbosity. + initialize_logger(log_verbosity, update_log_verbosity=True) while True: # API Authentication. @@ -617,10 +632,10 @@ def main(): # Get metric & statistics for vms and nodes. node_statistics = get_node_statistics(api_object, ignore_nodes) - vm_statistics = get_vm_statistics(api_object, ignore_vms) + vm_statistics = get_vm_statistics(api_object, ignore_vms) # Calculate rebalancing of vms. - node_statistics_rebalanced, vm_statistics_rebalanced = balancing_calculations(balancing_method, node_statistics, vm_statistics, balanciness) + node_statistics_rebalanced, vm_statistics_rebalanced = balancing_calculations(balancing_method, node_statistics, vm_statistics, balanciness, balancing_mode) # Rebalance vms to new nodes within the cluster. run_vm_rebalancing(api_object, vm_statistics_rebalanced, app_args) diff --git a/proxlb.conf b/proxlb.conf index 9983a3b..fc4c3d5 100644 --- a/proxlb.conf +++ b/proxlb.conf @@ -5,8 +5,10 @@ api_pass: FooBar verify_ssl: 1 [balancing] method: memory +mode: used ignore_nodes: dummynode01,dummynode02 ignore_vms: testvm01,testvm02 [service] daemon: 1 schedule: 24 +log_verbosity: CRITICAL