diff --git a/.gitignore b/.gitignore index c22521934..cec577a85 100644 --- a/.gitignore +++ b/.gitignore @@ -39,8 +39,9 @@ flux.out slurm*.out docs/build/ -# Tox files +# Test files .tox/* +.coverage # Jupyter jupyter/.ipynb_checkpoints diff --git a/CHANGELOG.md b/CHANGELOG.md index 21b4427b1..5bd534b1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.12.2b1] +## [1.12.2] ### Added - Conflict handler option to the `dict_deep_merge` function in `utils.py` - Ability to add module-specific pytest fixtures @@ -19,6 +19,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New step return code `$(MERLIN_RAISE_ERROR)` to force an error to be raised by a task (mainly for testing) - Added description of this to docs - New test to ensure a single failed task won't break a workflow +- Several new unit tests for the following subdirectories: + - `merlin/common/` + - `merlin/config/` + - `merlin/examples/` + - `merlin/server/` +- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures + - `RedisServerManager`: context to help with starting/stopping a redis server for tests + - `CeleryWorkersManager`: context to help with starting/stopping workers for tests +- Ability to copy and print the `Config` object from `merlin/config/__init__.py` +- Equality method to the `ContainerFormatConfig` and `ContainerConfig` objects from `merlin/server/server_util.py` ### Changed - `merlin info` is cleaner and gives python package info @@ -28,6 +38,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added fixtures for `merlin status` tests that copy the workspace to a temporary directory so you can see exactly what's run in a test - Batch block and workers now allow for variables to be used in node settings - Task id is now the path to the directory +- Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier +- Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier +- Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot` ### Fixed - Bugfix for output of `merlin example openfoam_wf_singularity` @@ -95,8 +108,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - this required adding a decent amount of test files to help with the tests; these can be found under the tests/unit/study/status_test_files directory - Pytest fixtures in the `conftest.py` file of the integration test suite - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command -- Tests for the `celeryadapter.py` module -- New CeleryTestWorkersManager context to help with starting/stopping workers for tests +- Coverage to the test suite. This includes adding tests for: + - `merlin/common/` + - `merlin/config/` + - `merlin/examples/` + - `celeryadapter.py` +- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures + - `RedisServerManager`: context to help with starting/stopping a redis server for tests + - `CeleryWorkersManager`: context to help with starting/stopping workers for tests +- Ability to copy and print the `Config` object from `merlin/config/__init__.py` ### Changed - Reformatted the entire `merlin status` command @@ -132,7 +152,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `merlin monitor` command will now keep an allocation up if the queues are empty and workers are still processing tasks - Add the restart keyword to the specification docs - Cyclical imports and config imports that could easily cause ci issues - ## [1.11.1] ### Fixed - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) diff --git a/Makefile b/Makefile index 08fb7d0f8..a261b1d99 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/config.mk b/config.mk index f1cfbcea3..1d6455ae3 100644 --- a/config.mk +++ b/config.mk @@ -18,7 +18,7 @@ else endif VER?=1.0.0 -VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+ +VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+\(b[0-9]\+\)\? YEAR=20[0-9][0-9] NEW_YEAR?=2023 CHANGELOG_VSTRING="## \[$(VSTRING)\]" diff --git a/merlin/__init__.py b/merlin/__init__.py index a2d173a8f..b6c065fd7 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.12.2b1" +__version__ = "1.12.2" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 5c90a4b12..bcada70af 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index eb10f1a12..81a17c5b2 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 57477ea1f..37cabcad1 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 57477ea1f..37cabcad1 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index a95ba0872..00d90232b 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/dumper.py b/merlin/common/dumper.py index 96a940357..5757b97ec 100644 --- a/merlin/common/dumper.py +++ b/merlin/common/dumper.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1 +# This file is part of Merlin, Version: 1.12.2 # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 16aa2f87c..790054c16 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 872699ae1..ba6e9dcc6 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index c7808bd3b..e52ded9c7 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -225,8 +225,8 @@ def __setitem__(self, full_address, sub_tree): # Replace if we already have something at this address. if delete_me is not None: - self.children.__delitem__(full_address) SampleIndex.check_valid_addresses_for_insertion(full_address, sub_tree) + self.children.__delitem__(full_address) self.children[full_address] = sub_tree return raise KeyError diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index eb4fbcc61..75de4676b 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 57477ea1f..37cabcad1 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index b1932cd28..e38b0ebbd 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -52,11 +52,10 @@ def _get_key_path(): except AttributeError: key_filepath = "~/.merlin/encrypt_data_key" - try: - key_filepath = os.path.abspath(os.path.expanduser(key_filepath)) - except KeyError as e: - raise ValueError("Error! No password provided for RabbitMQ") from e - return key_filepath + if key_filepath is None: + raise ValueError("Error! No password provided for RabbitMQ") + + return os.path.abspath(os.path.expanduser(key_filepath)) def _gen_key(key_path): diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index d597f084b..23bdaacf8 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 143d3bf12..736d92225 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 0a6c585cf..21c97c709 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -35,6 +35,7 @@ import numpy as np +# TODO should we move this to merlin-spellbook? def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False): """Scale samples to new limits, either log10 or linearly. diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 41645e249..f1451b4c4 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -31,7 +31,7 @@ """ Used to store the application configuration. """ - +from copy import copy from types import SimpleNamespace from typing import Dict, List, Optional @@ -56,6 +56,35 @@ def __init__(self, app_dict): self.results_backend: Optional[SimpleNamespace] self.load_app_into_namespaces(app_dict) + def __copy__(self): + """ + A magic method to allow this class to be copied with copy(instance_of_Config). + """ + cls = self.__class__ + result = cls.__new__(cls) + copied_attrs = { + "celery": copy(self.__dict__["celery"]), + "broker": copy(self.__dict__["broker"]), + "results_backend": copy(self.__dict__["results_backend"]), + } + result.__dict__.update(copied_attrs) + return result + + def __str__(self): + """ + A magic method so we can print the CONFIG class. + """ + formatted_str = "config:" + attrs = {"celery": self.celery, "broker": self.broker, "results_backend": self.results_backend} + for name, attr in attrs.items(): + if attr is not None: + items = (f" {k}: {v!r}" for k, v in attr.__dict__.items()) + joined_items = "\n".join(items) + formatted_str += f"\n {name}:\n{joined_items}" + else: + formatted_str += f"\n {name}:\n None" + return formatted_str + def load_app_into_namespaces(self, app_dict: Dict) -> None: """ Makes the application dictionary into a namespace, sets the attributes of the Config from the namespace values. diff --git a/merlin/config/broker.py b/merlin/config/broker.py index dc8131c28..6145691a5 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -85,13 +85,13 @@ def get_rabbit_connection(include_password, conn="amqps"): password_filepath = CONFIG.broker.password LOG.debug(f"Broker: password filepath = {password_filepath}") password_filepath = os.path.abspath(expanduser(password_filepath)) - except KeyError as e: # pylint: disable=C0103 - raise ValueError("Broker: No password provided for RabbitMQ") from e + except (AttributeError, KeyError) as exc: + raise ValueError("Broker: No password provided for RabbitMQ") from exc try: password = read_file(password_filepath) - except IOError as e: # pylint: disable=C0103 - raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from e + except IOError as exc: + raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from exc try: port = CONFIG.broker.port @@ -205,12 +205,6 @@ def get_connection_string(include_password=True): except AttributeError: broker = "" - try: - config_path = CONFIG.celery.certs - config_path = os.path.abspath(os.path.expanduser(config_path)) - except AttributeError: - config_path = None - if broker not in BROKERS: raise ValueError(f"Error: {broker} is not a supported broker.") return _sort_valid_broker(broker, include_password) diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 5794599dc..ff006c739 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 1634b833f..010641a3f 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 259e249a6..ae58f0340 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -236,6 +236,12 @@ def get_mysql(certs_path=None, mysql_certs=None, include_password=True): mysql_config["password"] = "******" mysql_config["server"] = server + # Ensure the ssl_key, ssl_ca, and ssl_cert keys are all set + if mysql_certs == MYSQL_CONFIG_FILENAMES: + for key, cert_file in mysql_certs.items(): + if key not in mysql_config: + mysql_config[key] = os.path.join(certs_path, cert_file) + return MYSQL_CONNECTION_STRING.format(**mysql_config) diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 46672ba1f..090398187 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -77,8 +77,14 @@ def get_priority(priority: Priority) -> int: :param priority: The priority value that we want :returns: The priority value as an integer """ - if priority not in Priority: - raise ValueError(f"Invalid priority: {priority}") + priority_err_msg = f"Invalid priority: {priority}" + try: + # In python 3.12+ if something is not in the enum it will just return False + if priority not in Priority: + raise ValueError(priority_err_msg) + # In python 3.11 and below, a TypeError is raised when looking for something in an enum that is not there + except TypeError: + raise ValueError(priority_err_msg) priority_map = determine_priority_map(CONFIG.broker.name.lower()) return priority_map.get(priority, priority_map[Priority.MID]) # Default to MID priority for unknown priorities diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 57477ea1f..37cabcad1 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index a1af0ac28..65818ce05 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 57477ea1f..37cabcad1 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 371471550..1b88783ff 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index a553d703b..bdb764b30 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -48,11 +48,19 @@ EXAMPLES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "workflows") +# TODO modify the example command to eliminate redundancy +# - e.g. running `merlin example flux_local` will produce the same output +# as running `merlin example flux_par` or `merlin example flux_par_restart`. +# This should just be `merlin example flux`. +# - restart and restart delay should be one example +# - feature demo and remote feature demo should be one example +# - all openfoam examples should just be under one openfoam label + def gather_example_dirs(): """Get all the example directories""" result = {} - for directory in os.listdir(EXAMPLES_DIR): + for directory in sorted(os.listdir(EXAMPLES_DIR)): result[directory] = directory return result @@ -82,7 +90,7 @@ def list_examples(): for example_dir in gather_example_dirs(): directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") specs = glob.glob(directory + "*.yaml") - for spec in specs: + for spec in sorted(specs): if "template" in spec: continue with open(spec) as f: # pylint: disable=C0103 diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 572135aec..158c6f073 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 6cd6a745a..55f5f37f8 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 4bb005985..6ca79e86d 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -425,6 +425,14 @@ def process_server(args: Namespace): Route to the correct function based on the command given via the CLI """ + try: + lc_all_val = os.environ["LC_ALL"] + if lc_all_val != "C": + raise ValueError(f"The 'LC_ALL' environment variable is currently set to {lc_all_val} but it must be set to 'C'.") + except KeyError: + LOG.debug("The 'LC_ALL' environment variable was not set. Setting this to 'C'.") + os.environ["LC_ALL"] = "C" # Necessary for Redis to configure LOCALE + if args.commands == "init": init_server() elif args.commands == "start": diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 5253c79e4..f82d814f6 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index d9114bbcd..552323260 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 522d67d1d..df2fc6e6a 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index be2b944a0..bb31d6b09 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -48,7 +48,7 @@ pull_server_config, pull_server_image, ) -from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers +from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers, ServerConfig LOG = logging.getLogger("merlin") @@ -70,17 +70,13 @@ def init_server() -> None: LOG.info("Merlin server initialization successful.") -# Pylint complains that there's too many branches in this function but -# it looks clean to me so we'll ignore it -def config_server(args: Namespace) -> None: # pylint: disable=R0912 +def apply_config_changes(server_config: ServerConfig, args: Namespace): """ - Process the merlin server config flags to make changes and edits to appropriate configurations - based on the input passed in by the user. + Apply any configuration changes that the user is requesting. + + :param server_config: An instance of ServerConfig containing all the necessary configuration values + :param args: An argumentparser namespace object with args from the user """ - server_config = pull_server_config() - if not server_config: - LOG.error('Try to run "merlin server init" again to reinitialize values.') - return False redis_config = RedisConfig(server_config.container.get_config_path()) redis_config.set_ip_address(args.ipaddress) @@ -98,9 +94,7 @@ def config_server(args: Namespace) -> None: # pylint: disable=R0912 redis_config.set_directory(args.directory) - redis_config.set_snapshot_seconds(args.snapshot_seconds) - - redis_config.set_snapshot_changes(args.snapshot_changes) + redis_config.set_snapshot(seconds=args.snapshot_seconds, changes=args.snapshot_changes) redis_config.set_snapshot_file(args.snapshot_file) @@ -116,14 +110,31 @@ def config_server(args: Namespace) -> None: # pylint: disable=R0912 else: LOG.info("Add changes to config file and exisiting containers.") - server_config = pull_server_config() - if not server_config: + +# Pylint complains that there's too many branches in this function but +# it looks clean to me so we'll ignore it +def config_server(args: Namespace) -> None: # pylint: disable=R0912 + """ + Process the merlin server config flags to make changes and edits to appropriate configurations + based on the input passed in by the user. + + :param args: An argumentparser namespace object with args from the user + """ + server_config_before_changes = pull_server_config() + if not server_config_before_changes: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + apply_config_changes(server_config_before_changes, args) + + server_config_after_changes = pull_server_config() + if not server_config_after_changes: LOG.error('Try to run "merlin server init" again to reinitialize values.') return False # Read the user from the list of avaliable users - redis_users = RedisUsers(server_config.container.get_user_file_path()) - redis_config = RedisConfig(server_config.container.get_config_path()) + redis_users = RedisUsers(server_config_after_changes.container.get_user_file_path()) + redis_config = RedisConfig(server_config_after_changes.container.get_config_path()) if args.add_user is not None: # Log the user in a file @@ -157,7 +168,7 @@ def status_server() -> None: Get the server status of the any current running containers for merlin server """ current_status = get_server_status() - if current_status == ServerStatus.NOT_INITALIZED: + if current_status == ServerStatus.NOT_INITIALIZED: LOG.info("Merlin server has not been initialized.") LOG.info("Please initalize server by running 'merlin server init'") elif current_status == ServerStatus.MISSING_CONTAINER: @@ -169,15 +180,17 @@ def status_server() -> None: LOG.info("Merlin server is running.") -def start_server() -> bool: # pylint: disable=R0911 +def check_for_not_running_server() -> bool: """ - Start a merlin server container using singularity. - :return:: True if server was successful started and False if failed. + When starting a server the status must be NOT_RUNNING. If it's any other + status we need to log an error for the user to see. + + :returns: True if the status is NOT_RUNNING. False otherwise. """ current_status = get_server_status() uninitialized_err = "Merlin server has not been intitialized. Please run 'merlin server init' first." status_errors = { - ServerStatus.NOT_INITALIZED: uninitialized_err, + ServerStatus.NOT_INITIALIZED: uninitialized_err, ServerStatus.MISSING_CONTAINER: uninitialized_err, ServerStatus.RUNNING: """Merlin server already running. Stop current server with 'merlin server stop' before attempting to start a new server.""", @@ -187,11 +200,16 @@ def start_server() -> bool: # pylint: disable=R0911 LOG.info(status_errors[current_status]) return False - server_config = pull_server_config() - if not server_config: - LOG.error('Try to run "merlin server init" again to reinitialize values.') - return False + return True + + +def start_container(server_config: ServerConfig) -> subprocess.Popen: + """ + Given a server configuration, use it to start up a container. + :param server_config: The ServerConfig instance that holds information about the server to start. + :returns: A subprocess started with subprocess.Popen that's executing the command to start the container. + """ image_path = server_config.container.get_image_path() config_path = server_config.container.get_config_path() path_errors = { @@ -202,7 +220,7 @@ def start_server() -> bool: # pylint: disable=R0911 for path in (image_path, config_path): if not os.path.exists(path): LOG.error(f"Unable to find {path_errors[path]} at {path}") - return False + return None # Pylint wants us to use with here but we don't need that process = subprocess.Popen( # pylint: disable=R1732 @@ -222,6 +240,17 @@ def start_server() -> bool: # pylint: disable=R0911 time.sleep(1) + return process + + +def server_started(process: subprocess.Popen, server_config: ServerConfig) -> bool: + """ + Check that the server spun up by `start_container` was started properly. + + :param process: The subprocess that was started by `start_container` + :param server_config: The ServerConfig instance that holds information about the redis server to start + :returns: True if the server started properly. False otherwise. + """ redis_start, redis_out = parse_redis_output(process.stdout) if not redis_start: @@ -243,6 +272,28 @@ def start_server() -> bool: # pylint: disable=R0911 LOG.info(f"Server started with PID {str(process.pid)}.") LOG.info(f'Merlin server operating on "{redis_out["hostname"]}" and port "{redis_out["port"]}".') + return True + + +def start_server() -> bool: # pylint: disable=R0911 + """ + Start a merlin server container using singularity. + :return:: True if server was successful started and False if failed. + """ + if not check_for_not_running_server(): + return False + + server_config = pull_server_config() + if not server_config: + LOG.error('Try to run "merlin server init" again to reinitialize values.') + return False + + process = start_container(server_config) + if process is None: + return False + + if not server_started(process, server_config): + return False redis_users = RedisUsers(server_config.container.get_user_file_path()) redis_config = RedisConfig(server_config.container.get_config_path()) diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index f58c7567a..36c2165bb 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -77,7 +77,7 @@ class ServerStatus(enum.Enum): """ RUNNING = 0 - NOT_INITALIZED = 1 + NOT_INITIALIZED = 1 MISSING_CONTAINER = 2 NOT_RUNNING = 3 ERROR = 4 @@ -92,8 +92,8 @@ def generate_password(length, pass_command: str = None) -> str: :return:: string value with given length """ if pass_command: - process = subprocess.run(pass_command.split(), shell=True, stdout=subprocess.PIPE) - return process.stdout + process = subprocess.run(pass_command, shell=True, capture_output=True, text=True) + return process.stdout.strip() characters = list(string.ascii_letters + string.digits + "!@#$%^&*()") @@ -119,7 +119,7 @@ def parse_redis_output(redis_stdout: BufferedReader) -> Tuple[bool, str]: server_init = False redis_config = {} line = redis_stdout.readline() - while line != "" or line is not None: + while line != b"" and line is not None: if not server_init: values = [ln for ln in line.split() if b"=" in ln] for val in values: @@ -136,6 +136,31 @@ def parse_redis_output(redis_stdout: BufferedReader) -> Tuple[bool, str]: return False, "Reached end of redis output without seeing 'Ready to accept connections'" +def copy_container_command_files(config_dir: str) -> bool: + """ + Copy the yaml files that contain instructions on how to run certain commands + for each container type to the config directory. + + :param config_dir: The path to the configuration dir where we'll copy files. + :returns: True if successful. False otherwise. + """ + files = [i + ".yaml" for i in CONTAINER_TYPES] + for file in files: + file_path = os.path.join(config_dir, file) + if os.path.exists(file_path): + LOG.info(f"{file} already exists.") + continue + LOG.info(f"Copying file {file} to configuration directory.") + try: + with resources.path("merlin.server", file) as config_file: + with open(file_path, "w") as outfile, open(config_file, "r") as infile: + outfile.write(infile.read()) + except OSError: + LOG.error(f"Destination location {config_dir} is not writable.") + return False + return True + + def create_server_config() -> bool: """ Create main configuration file for merlin server in the @@ -158,20 +183,8 @@ def create_server_config() -> bool: LOG.error(err) return False - files = [i + ".yaml" for i in CONTAINER_TYPES] - for file in files: - file_path = os.path.join(config_dir, file) - if os.path.exists(file_path): - LOG.info(f"{file} already exists.") - continue - LOG.info(f"Copying file {file} to configuration directory.") - try: - with resources.path("merlin.server", file) as config_file: - with open(file_path, "w") as outfile, open(config_file, "r") as infile: - outfile.write(infile.read()) - except OSError: - LOG.error(f"Destination location {config_dir} is not writable.") - return False + if not copy_container_command_files(config_dir): + return False # Load Merlin Server Configuration and apply it to app.yaml with resources.path("merlin.server", MERLIN_SERVER_CONFIG) as merlin_server_config: @@ -209,9 +222,6 @@ def config_merlin_server(): if os.path.exists(pass_file): LOG.info("Password file already exists. Skipping password generation step.") else: - # if "pass_command" in server_config["container"]: - # password = generate_password(PASSWORD_LENGTH, server_config["container"]["pass_command"]) - # else: password = generate_password(PASSWORD_LENGTH) with open(pass_file, "w+") as f: # pylint: disable=C0103 @@ -287,7 +297,7 @@ def pull_server_image() -> bool: """ Fetch the server image using singularity. - :return:: True if success and False if fail + :return: True if success and False if fail """ server_config = pull_server_config() if not server_config: @@ -318,8 +328,8 @@ def pull_server_image() -> bool: with resources.path("merlin.server", config_file) as file: with open(os.path.join(config_dir, config_file), "w") as outfile, open(file, "r") as infile: outfile.write(infile.read()) - except OSError: - LOG.error(f"Destination location {config_dir} is not writable.") + except OSError as exc: + LOG.error(f"Destination location {config_dir} is not writable. Raised from:\n{exc}") return False else: LOG.info("Redis configuration file already exist.") @@ -339,10 +349,10 @@ def get_server_status(): """ server_config = pull_server_config() if not server_config: - return ServerStatus.NOT_INITALIZED + return ServerStatus.NOT_INITIALIZED if not os.path.exists(server_config.container.get_config_dir()): - return ServerStatus.NOT_INITALIZED + return ServerStatus.NOT_INITIALIZED if not os.path.exists(server_config.container.get_image_path()): return ServerStatus.MISSING_CONTAINER diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index aa7c2765b..15f591264 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -60,7 +60,7 @@ def valid_ipv4(ip: str) -> bool: # pylint: disable=C0103 return False for i in arr: - if int(i) < 0 and int(i) > 255: + if int(i) < 0 or int(i) > 255: return False return True @@ -121,6 +121,15 @@ def __init__(self, data: dict) -> None: self.pass_file = data["pass_file"] if "pass_file" in data else self.PASSWORD_FILE self.user_file = data["user_file"] if "user_file" in data else self.USERS_FILE + def __eq__(self, other: "ContainerFormatConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ContainerFormatConfig object to check if they're the same + """ + variables = ("format", "image_type", "image", "url", "config", "config_dir", "pfile", "pass_file", "user_file") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_format(self) -> str: """Getter method to get the container format""" return self.format @@ -208,6 +217,15 @@ def __init__(self, data: dict) -> None: self.stop_command = data["stop_command"] if "stop_command" in data else self.STOP_COMMAND self.pull_command = data["pull_command"] if "pull_command" in data else self.PULL_COMMAND + def __eq__(self, other: "ContainerFormatConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ContainerFormatConfig object to check if they're the same + """ + variables = ("command", "run_command", "stop_command", "pull_command") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_command(self) -> str: """Getter method to get the container command""" return self.command @@ -242,6 +260,15 @@ def __init__(self, data: dict) -> None: self.status = data["status"] if "status" in data else self.STATUS_COMMAND self.kill = data["kill"] if "kill" in data else self.KILL_COMMAND + def __eq__(self, other: "ProcessConfig"): + """ + Equality magic method used for testing this class + + :param other: Another ProcessConfig object to check if they're the same + """ + variables = ("status", "kill") + return all(getattr(self, attr) == getattr(other, attr) for attr in variables) + def get_status_command(self) -> str: """Getter method to get the status command""" return self.status @@ -264,12 +291,10 @@ class ServerConfig: # pylint: disable=R0903 container_format: ContainerFormatConfig = None def __init__(self, data: dict) -> None: - if "container" in data: - self.container = ContainerConfig(data["container"]) - if "process" in data: - self.process = ProcessConfig(data["process"]) - if self.container.get_format() in data: - self.container_format = ContainerFormatConfig(data[self.container.get_format()]) + self.container = ContainerConfig(data["container"]) if "container" in data else None + self.process = ProcessConfig(data["process"]) if "process" in data else None + container_format_data = data.get(self.container.get_format() if self.container else None) + self.container_format = ContainerFormatConfig(container_format_data) if container_format_data else None class RedisConfig: @@ -279,16 +304,14 @@ class RedisConfig: to write those changes into a redis readable config file. """ - filename = "" - entry_order = [] - entries = {} - comments = {} - trailing_comments = "" - changed = False - def __init__(self, filename) -> None: self.filename = filename self.changed = False + self.entry_order = [] + self.entries = {} + self.comments = {} + self.trailing_comments = "" + self.changed = False self.parse() def parse(self) -> None: @@ -368,7 +391,7 @@ def get_port(self) -> str: """Getter method to get the port from the redis config""" return self.get_config_value("port") - def set_port(self, port: str) -> bool: + def set_port(self, port: int) -> bool: """Validates and sets a given port""" if port is None: return False @@ -403,59 +426,56 @@ def set_directory(self, directory: str) -> bool: """ if directory is None: return False + # Create the directory if it doesn't exist if not os.path.exists(directory): os.mkdir(directory) LOG.info(f"Created directory {directory}") - # Validate the directory input - if os.path.exists(directory): - # Set the save directory to the redis config - if not self.set_config_value("dir", directory): - LOG.error("Unable to set directory for redis config") - return False - else: - LOG.error(f"Directory {directory} given does not exist and could not be created.") + # Set the save directory to the redis config + if not self.set_config_value("dir", directory): + LOG.error("Unable to set directory for redis config") return False LOG.info(f"Directory is set to {directory}") return True - def set_snapshot_seconds(self, seconds: int) -> bool: - """Sets the snapshot wait time""" - if seconds is None: - return False - # Set the snapshot second in the redis config - value = self.get_config_value("save") - if value is None: - LOG.error("Unable to get exisiting parameter values for snapshot") - return False + def set_snapshot(self, seconds: int = None, changes: int = None) -> bool: + """ + Sets the 'seconds' and/or 'changes' values of the snapshot setting, + depending on what the user requests. + + :param seconds: The first value of snapshot to change. If we're leaving it the + same this will be None. + :param changes: The second value of snapshot to change. If we're leaving it the + same this will be None. + :returns: True if successful, False otherwise. + """ - value = value.split() - value[0] = str(seconds) - value = " ".join(value) - if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") + # If both values are None, this method is doing nothing + if seconds is None and changes is None: return False - LOG.info(f"Snapshot wait time is set to {seconds} seconds") - return True - - def set_snapshot_changes(self, changes: int) -> bool: - """Sets the snapshot threshold""" - if changes is None: - return False - # Set the snapshot changes into the redis config + # Grab the snapshot value from the redis config value = self.get_config_value("save") if value is None: LOG.error("Unable to get exisiting parameter values for snapshot") return False + # Update the snapshot value value = value.split() - value[1] = str(changes) + log_msg = "" + if seconds is not None: + value[0] = str(seconds) + log_msg += f"Snapshot wait time is set to {seconds} seconds. " + if changes is not None: + value[1] = str(changes) + log_msg += f"Snapshot threshold is set to {changes} changes." value = " ".join(value) + + # Set the new snapshot value if not self.set_config_value("save", value): - LOG.error("Unable to set snapshot value seconds") + LOG.error("Unable to set snapshot value") return False - LOG.info(f"Snapshot threshold is set to {changes} changes") + LOG.info(log_msg) return True def set_snapshot_file(self, file: str) -> bool: @@ -483,7 +503,7 @@ def set_append_mode(self, mode: str) -> bool: LOG.error("Unable to set append_mode in redis config") return False else: - LOG.error("Not a valid append_mode(Only valid modes are always, everysec, no)") + LOG.error("Not a valid append_mode (Only valid modes are always, everysec, no)") return False LOG.info(f"Append mode is set to {mode}") @@ -603,7 +623,7 @@ def set_password(self, user: str, password: str): self.users[user].set_password(password) return True - def remove_user(self, user) -> bool: + def remove_user(self, user: str) -> bool: """Remove a user from the dict of users""" if user in self.users: del self.users[user] diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 57477ea1f..37cabcad1 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index fbb70f8d7..706111851 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 6c9bd9c09..1aa700607 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index ac514a369..b9f9eee65 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 316e76f86..049d8030f 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 5ffffd959..141ebf297 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 57477ea1f..37cabcad1 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 16482f399..cf2c53116 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 5b5bdd419..60bded1b2 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index c1b9dff78..06f75beac 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 6380b1a9b..b76eb2172 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/status.py b/merlin/study/status.py index fbeb4d46d..b785ee140 100644 --- a/merlin/study/status.py +++ b/merlin/study/status.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1 +# This file is part of Merlin, Version: 1.12.2 # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/status_constants.py b/merlin/study/status_constants.py index b7dfe7fa3..78ce331a6 100644 --- a/merlin/study/status_constants.py +++ b/merlin/study/status_constants.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1 +# This file is part of Merlin, Version: 1.12.2 # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/status_renderers.py b/merlin/study/status_renderers.py index 02d6ab948..2100c2bda 100644 --- a/merlin/study/status_renderers.py +++ b/merlin/study/status_renderers.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1 +# This file is part of Merlin, Version: 1.12.2 # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 26d737e14..8e0356b68 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index f30e36058..1d7b9d96d 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 2e69f5779..50b3474e2 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/requirements/dev.txt b/requirements/dev.txt index 3695c6164..ab5962119 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,6 +5,7 @@ dep-license flake8 isort pytest +pytest-cov pylint twine sphinx>=2.0.0 diff --git a/setup.cfg b/setup.cfg index a000df59a..77ac2d84f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,3 +26,9 @@ max-line-length = 127 files=best_practices,test ignore_missing_imports=true + +[coverage:run] +omit = + merlin/ascii_art.py + merlin/config/celeryconfig.py + merlin/examples/examples.py diff --git a/setup.py b/setup.py index 0ee113e0a..48def98f6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/README.md b/tests/README.md index 22efc5470..9b2f7ba1f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,17 +1,21 @@ # Tests This directory utilizes pytest to create and run our test suite. -Here we use pytest fixtures to create a local redis server and a celery app for testing. This directory is organized like so: -- `conftest.py` - The script containing all fixtures for our tests -- `unit/` - The directory containing unit tests - - `test_*.py` - The actual test scripts to run +- `conftest.py` - The script containing common fixtures for our tests +- `context_managers/` - The directory containing context managers used for testing + - `celery_workers_manager.py` - A context manager used to manage celery workers for integration testing + - `server_manager.py` - A context manager used to manage the redis server used for integration testing +- `fixtures/` - The directory containing specific test module fixtures + - `.py` - Fixtures for specific test modules - `integration/` - The directory containing integration tests - `definitions.py` - The test definitions - `run_tests.py` - The script to run the tests defined in `definitions.py` - `conditions.py` - The conditions to test against +- `unit/` - The directory containing unit tests + - `test_*.py` - The actual test scripts to run ## How to Run @@ -44,6 +48,28 @@ To run one unique test: python -m pytest /path/to/test_specific_file.py::TestCertainClass::test_unique_test ``` +## Viewing Results + +Test results will be written to `/tmp/$(whoami)/pytest-of-$(whoami)/pytest-current/python_{major}.{minor}.{micro}_current/`. + +It's good practice to set up a subdirectory in this temporary output folder for each module that you're testing. You can see an example of how this is set up in the files within the module-specific fixture directory. For instance, you can see this in the `examples_testing_dir` fixture from the `tests/fixtures/examples.py` file: + +``` +@pytest.fixture(scope="session") +def examples_testing_dir(temp_output_dir: str) -> str: + """ + Fixture to create a temporary output directory for tests related to the examples functionality. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for examples tests + """ + testing_dir = f"{temp_output_dir}/examples_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir +``` + ## Killing the Test Server In case of an issue with the test suite, or if you stop the tests with `ctrl+C`, you may need to stop @@ -58,58 +84,45 @@ not connected> quit ## The Fixture Process Explained -In the world of pytest testing, fixtures are like the building blocks that create a sturdy foundation for your tests. -They ensure that every test starts from the same fresh ground, leading to reliable and consistent results. This section -will dive into the nitty-gritty of these fixtures, showing you how they're architected in this test suite, how to use -them in your tests here, how to combine them for more complex scenarios, how long they stick around during testing, and -what it means to yield a fixture. +In the world of pytest testing, fixtures are like the building blocks that create a sturdy foundation for your tests. They ensure that every test starts from the same fresh ground, leading to reliable and consistent results. This section will dive into the nitty-gritty of these fixtures, showing you how they're architected in this test suite, how to use them in your tests here, how to combine them for more complex scenarios, how long they stick around during testing, and what it means to yield a fixture. ### Fixture Architecture -Fixtures can be defined in two locations: +Fixtures can be defined in two locations within this test suite: -1. `tests/conftest.py`: This file located at the root of the test suite houses common fixtures that are utilized -across various test modules -2. `tests/fixtures/`: This directory contains specific test module fixtures. Each fixture file is named according -to the module(s) that the fixtures defined within are for. +1. `tests/conftest.py`: This file located at the root of the test suite houses common fixtures that are utilized across various test modules +2. `tests/fixtures/`: This directory contains specific test module fixtures. Each fixture file is named according to the module(s) that the fixtures defined within are for. Credit for this setup must be given to [this Medium article](https://medium.com/@nicolaikozel/modularizing-pytest-fixtures-fd40315c5a93). #### Fixture Naming Conventions -For fixtures defined within the `tests/fixtures/` directory, the fixture name should be prefixed by the name of the -fixture file they are defined in. +For fixtures defined within the `tests/fixtures/` directory, the fixture name should be prefixed by the name of the fixture file they are defined in. #### Importing Fixtures as Plugins -Fixtures located in the `tests/fixtures/` directory are technically plugins. Therefore, to use them we must -register them as plugins within the `conftest.py` file (see the top of said file for the implementation). -This allows them to be discovered and used by test modules throughout the suite. +Fixtures located in the `tests/fixtures/` directory are technically plugins. Therefore, to use them we must register them as plugins within the `conftest.py` file (see the top of said file for the implementation). This allows them to be discovered and used by test modules throughout the suite. -**You do not have to register the fixtures you define as plugins in `conftest.py` since the registration there -uses `glob` to grab everything from the `tests/fixtures/` directory automatically.** +**You do not have to register the fixtures you define as plugins in `conftest.py` since the registration there uses `glob` to grab everything from the `tests/fixtures/` directory automatically.** ### How to Integrate Fixtures Into Tests -Probably the most important part of fixtures is understanding how to use them. Luckily, this process is very -simple and can be dumbed down to just a couple steps: +Probably the most important part of fixtures is understanding how to use them. Luckily, this process is very simple and can be dumbed down to just a couple steps: -0. **[Module-specific fixtures only]** If you're creating a module-specific fixture (i.e. a fixture that won't be used throughout the entire test -suite), then create a file in the `tests/fixtures/` directory. +0. **[Module-specific fixtures only]** If you're creating a module-specific fixture (i.e. a fixture that won't be used throughout the entire test suite), then create a file in the `tests/fixtures/` directory. -1. Create a fixture in either the `conftest.py` file or the file you created in the `tests/fixtures/` directory -by using the `@pytest.fixture` decorator. For example: +1. Create a fixture in either the `conftest.py` file or the file you created in the `tests/fixtures/` directory by using the `@pytest.fixture` decorator. For example: ``` @pytest.fixture -def dummy_fixture(): +def dummy_fixture() -> str: return "hello world" ``` 2. Use it as an argument in a test function (you don't even need to import it!): ``` -def test_dummy(dummy_fixture): +def test_dummy(dummy_fixture: str): assert dummy_fixture == "hello world" ``` @@ -117,22 +130,18 @@ For more information, see [Pytest's documentation](https://docs.pytest.org/en/7. ### Fixtureception -One of the coolest and most useful aspects of fixtures that we utilize in this test suite is the ability for -fixtures to be used within other fixtures. For more info on this from pytest, see -[here](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#fixtures-can-request-other-fixtures). +One of the coolest and most useful aspects of fixtures that we utilize in this test suite is the ability for fixtures to be used within other fixtures. For more info on this from pytest, see [here](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#fixtures-can-request-other-fixtures). + +Pytest will handle fixtures within fixtures in a stack-based way. Let's look at how creating the `redis_pass` fixture from our `conftest.py` file works in order to illustrate the process. -Pytest will handle fixtures within fixtures in a stack-based way. Let's look at how creating the `redis_pass` -fixture from our `conftest.py` file works in order to illustrate the process. -1. First, we start by telling pytest that we want to use the `redis_pass` fixture by providing it as an argument -to a test/fixture: +1. First, we start by telling pytest that we want to use the `redis_pass` fixture by providing it as an argument to a test/fixture: ``` def test_example(redis_pass): ... ``` -2. Now pytest will find the `redis_pass` fixture and put it at the top of the stack to be created. However, -it'll see that this fixture requires another fixture `merlin_server_dir` as an argument: +2. Now pytest will find the `redis_pass` fixture and put it at the top of the stack to be created. However, it'll see that this fixture requires another fixture `merlin_server_dir` as an argument: ``` @pytest.fixture(scope="session") @@ -140,8 +149,7 @@ def redis_pass(merlin_server_dir): ... ``` -3. Pytest then puts the `merlin_server_dir` fixture at the top of the stack, but similarly it sees that this fixture -requires yet another fixture `temp_output_dir`: +3. Pytest then puts the `merlin_server_dir` fixture at the top of the stack, but similarly it sees that this fixture requires yet another fixture `temp_output_dir`: ``` @pytest.fixture(scope="session") @@ -149,34 +157,24 @@ def merlin_server_dir(temp_output_dir: str) -> str: ... ``` -4. This process continues until it reaches a fixture that doesn't require any more fixtures. At this point the base -fixture is created and pytest will start working its way back up the stack to the first fixture it looked at (in this -case `redis_pass`). +4. This process continues until it reaches a fixture that doesn't require any more fixtures. At this point the base fixture is created and pytest will start working its way back up the stack to the first fixture it looked at (in this case `redis_pass`). -5. Once all required fixtures are created, execution will be returned to the test which can now access the fixture -that was requested (`redis_pass`). +5. Once all required fixtures are created, execution will be returned to the test which can now access the fixture that was requested (`redis_pass`). -As you can see, if we have to re-do this process for every test it could get pretty time intensive. This is where fixture -scopes come to save the day. +As you can see, if we have to re-do this process for every test it could get pretty time intensive. This is where fixture scopes come to save the day. ### Fixture Scopes -There are several different scopes that you can set for fixtures. The majority of our fixtures in `conftest.py` -use a `session` scope so that we only have to create the fixtures one time (as some of them can take a few seconds -to set up). The goal is to create fixtures with the most general use-case in mind so that we can re-use them for -larger scopes, which helps with efficiency. +There are several different scopes that you can set for fixtures. The majority of our fixtures in `conftest.py` use a `session` scope so that we only have to create the fixtures one time (as some of them can take a few seconds to set up). The goal for fixtures defined in `conftest.py` is to create fixtures with the most general use-case in mind so that we can re-use them for larger scopes, which helps with efficiency. + +For fixtures that need to be reset on each run, we generally try to place these in the module-specific fixture directory `tests/fixtures/`. -For more info on scopes, see -[Pytest's Fixture Scope documentation](https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session). +For more info on scopes, see [Pytest's Fixture Scope documentation](https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session). ### Yielding Fixtures -In several fixtures throughout our test suite, we need to run some sort of teardown for the fixture. For example, -once we no longer need the `redis_server` fixture, we need to shut the server down so it stops using resources. -This is where yielding fixtures becomes extremely useful. +In several fixtures throughout our test suite, we need to run some sort of teardown for the fixture. For example, once we no longer need the `redis_server` fixture, we need to shut the server down so it stops using resources. This is where yielding fixtures becomes extremely useful. -Using the `yield` keyword allows execution to be returned to a test that needs the fixture once the feature has -been set up. After all tests using the fixture have been ran, execution will return to the fixture for us to run -our teardown code. +Using the `yield` keyword allows execution to be returned to a test that needs the fixture once the feature has been set up. After all tests using the fixture have been ran, execution will return to the fixture for us to run our teardown code. For more information on yielding fixtures, see [Pytest's documentation](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#teardown-cleanup-aka-fixture-finalization). \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index bea07f64c..eb68542bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -28,42 +28,75 @@ # SOFTWARE. ############################################################################### """ -This module contains pytest fixtures to be used throughout the entire -integration test suite. +This module contains pytest fixtures to be used throughout the entire test suite. """ import os -import subprocess +import sys +from copy import copy from glob import glob from time import sleep from typing import Dict import pytest -import redis +import yaml from _pytest.tmpdir import TempPathFactory from celery import Celery from celery.canvas import Signature -from tests.celery_test_workers import CeleryTestWorkersManager +from merlin.config.configfile import CONFIG +from tests.constants import CERT_FILES, SERVER_PASS +from tests.context_managers.celery_workers_manager import CeleryWorkersManager +from tests.context_managers.server_manager import RedisServerManager +from tests.utils import create_cert_files, create_pass_file + + +# pylint: disable=redefined-outer-name ####################################### # Loading in Module Specific Fixtures # ####################################### + + pytest_plugins = [ fixture_file.replace("/", ".").replace(".py", "") for fixture_file in glob("tests/fixtures/[!__]*.py", recursive=True) ] -class RedisServerError(Exception): - """ - Exception to signal that the server wasn't pinged properly. - """ +####################################### +#### Helper Functions for Fixtures #### +####################################### -class ServerInitError(Exception): +def create_encryption_file(key_filepath: str, encryption_key: bytes, app_yaml_filepath: str = None): """ - Exception to signal that there was an error initializing the server. + Check if an encryption file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the encryption key to the file. If an app.yaml + filepath has been passed to this function then we'll need to update it so that the encryption + key points to the `key_filepath`. + + :param key_filepath: The path to the file that will store our encryption key + :param encryption_key: An encryption key to be used for testing + :param app_yaml_filepath: A path to the app.yaml file that needs to be updated """ + if not os.path.exists(key_filepath): + with open(key_filepath, "w") as key_file: + key_file.write(encryption_key.decode("utf-8")) + + if app_yaml_filepath is not None: + # Load up the app.yaml that was created by starting the server + with open(app_yaml_filepath, "r") as app_yaml_file: + app_yaml = yaml.load(app_yaml_file, yaml.Loader) + + # Modify the path to the encryption key and then save it + app_yaml["results_backend"]["encryption_key"] = key_filepath + with open(app_yaml_filepath, "w") as app_yaml_file: + yaml.dump(app_yaml, app_yaml_file) + + +####################################### +######### Fixture Definitions ######### +####################################### @pytest.fixture(scope="session") @@ -78,7 +111,9 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: """ # Log the cwd, then create and move into the temporary one cwd = os.getcwd() - temp_integration_outfile_dir = tmp_path_factory.mktemp("integration_outfiles_") + temp_integration_outfile_dir = tmp_path_factory.mktemp( + f"python_{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}_" + ) os.chdir(temp_integration_outfile_dir) yield temp_integration_outfile_dir @@ -88,77 +123,42 @@ def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: @pytest.fixture(scope="session") -def redis_pass() -> str: +def merlin_server_dir(temp_output_dir: str) -> str: """ - This fixture represents the password to the merlin test server. - - :returns: The redis password for our test server - """ - return "merlin-test-server" - - -@pytest.fixture(scope="session") -def merlin_server_dir(temp_output_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name - """ - This fixture will initialize the merlin server (i.e. create all the files we'll - need to start up a local redis server). It will return the path to the directory - containing the files needed for the server to start up. + The path to the merlin_server directory that will be created by the `redis_server` fixture. :param temp_output_dir: The path to the temporary output directory we'll be using for this test run - :param redis_pass: The password to the test redis server that we'll create here - :returns: The path to the merlin_server directory with the server configurations + :returns: The path to the merlin_server directory that will be created by the `redis_server` fixture """ - # Initialize the setup for the local redis server - # We'll also set the password to 'merlin-test-server' so it'll be easy to shutdown if there's an issue - subprocess.run(f"merlin server init; merlin server config -pwd {redis_pass}", shell=True, capture_output=True, text=True) - - # Check that the merlin server was initialized properly server_dir = f"{temp_output_dir}/merlin_server" if not os.path.exists(server_dir): - raise ServerInitError("The merlin server was not initialized properly.") - + os.mkdir(server_dir) return server_dir @pytest.fixture(scope="session") -def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name,unused-argument +def redis_server(merlin_server_dir: str, test_encryption_key: bytes) -> str: """ Start a redis server instance that runs on localhost:6379. This will yield the redis server uri that can be used to create a connection with celery. - :param merlin_server_dir: The directory to the merlin test server configuration. - This will not be used here but we need the server configurations before we can - start the server. - :param redis_pass: The raw redis password stored in the redis.pass file + :param merlin_server_dir: The directory to the merlin test server configuration + :param test_encryption_key: An encryption key to be used for testing :yields: The local redis server uri """ - # Start the local redis server - try: - # Need to set LC_ALL='C' before starting the server or else redis causes a failure - subprocess.run("export LC_ALL='C'; merlin server start", shell=True, timeout=5) - except subprocess.TimeoutExpired: - pass - - # Ensure the server started properly - host = "localhost" - port = 6379 - database = 0 - username = "default" - redis_client = redis.Redis(host=host, port=port, db=database, password=redis_pass, username=username) - if not redis_client.ping(): - raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") - - # Hand over the redis server url to any other fixtures/tests that need it - redis_server_uri = f"redis://{username}:{redis_pass}@{host}:{port}/{database}" - yield redis_server_uri - - # Kill the server; don't run this until all tests are done (accomplished with 'yield' above) - kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) - assert "Merlin server terminated." in kill_process.stderr + with RedisServerManager(merlin_server_dir, SERVER_PASS) as redis_server_manager: + redis_server_manager.initialize_server() + redis_server_manager.start_server() + create_encryption_file( + f"{merlin_server_dir}/encrypt_data_key", test_encryption_key, app_yaml_filepath=f"{merlin_server_dir}/app.yaml" + ) + # Yield the redis_server uri to any fixtures/tests that may need it + yield redis_server_manager.redis_server_uri + # The server will be stopped once this context reaches the end of it's execution here @pytest.fixture(scope="session") -def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer-name +def celery_app(redis_server: str) -> Celery: """ Create the celery app to be used throughout our integration tests. @@ -169,7 +169,7 @@ def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer- @pytest.fixture(scope="session") -def sleep_sig(celery_app: Celery) -> Signature: # pylint: disable=redefined-outer-name +def sleep_sig(celery_app: Celery) -> Signature: """ Create a task registered to our celery app and return a signature for it. Once requested by a test, you can set the queue you'd like to send this to @@ -201,7 +201,7 @@ def worker_queue_map() -> Dict[str, str]: @pytest.fixture(scope="class") -def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pylint: disable=redefined-outer-name +def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): """ Launch the workers on the celery app fixture using the worker and queue names defined in the worker_queue_map fixture. @@ -213,6 +213,171 @@ def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pyl # (basically just add in concurrency value to worker_queue_map) worker_info = {worker_name: {"concurrency": 1, "queues": [queue]} for worker_name, queue in worker_queue_map.items()} - with CeleryTestWorkersManager(celery_app) as workers_manager: + with CeleryWorkersManager(celery_app) as workers_manager: workers_manager.launch_workers(worker_info) yield + + +@pytest.fixture(scope="session") +def test_encryption_key() -> bytes: + """ + An encryption key to be used for tests that need it. + + :returns: The test encryption key + """ + return b"Q3vLp07Ljm60ahfU9HwOOnfgGY91lSrUmqcTiP0v9i0=" + + +####################################### +########### CONFIG Fixtures ########### +####################################### +# These are intended to be used # +# either by themselves or together # +# For example, you can use a rabbit # +# broker config and a redis results # +# backend config together # +####################################### +############ !!!WARNING!!! ############ +# DO NOT USE THE `config` FIXTURE # +# IN A TEST; IT HAS UNSET VALUES # +####################################### + + +@pytest.fixture(scope="function") +def config(merlin_server_dir: str, test_encryption_key: bytes): + """ + DO NOT USE THIS FIXTURE IN A TEST, USE `redis_config` OR `rabbit_config` INSTEAD. + This fixture is intended to be used strictly by the `redis_config` and `rabbit_config` + fixtures. It sets up the CONFIG object but leaves certain broker settings unset. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param test_encryption_key: An encryption key to be used for testing + """ + + # Create a copy of the CONFIG option so we can reset it after the test + orig_config = copy(CONFIG) + + # Create an encryption key file (if it doesn't already exist) + key_file = f"{merlin_server_dir}/encrypt_data_key" + create_encryption_file(key_file, test_encryption_key) + + # Set the broker configuration for testing + CONFIG.broker.password = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.port = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.name = None # This will be updated in `redis_broker_config` or `rabbit_broker_config` + CONFIG.broker.server = "127.0.0.1" + CONFIG.broker.username = "default" + CONFIG.broker.vhost = "host4testing" + CONFIG.broker.cert_reqs = "none" + + # Set the results_backend configuration for testing + CONFIG.results_backend.password = ( + None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + ) + CONFIG.results_backend.port = None # This will be updated in `redis_results_backend_config` + CONFIG.results_backend.name = ( + None # This will be updated in `redis_results_backend_config` or `mysql_results_backend_config` + ) + CONFIG.results_backend.dbname = None # This will be updated in `mysql_results_backend_config` + CONFIG.results_backend.server = "127.0.0.1" + CONFIG.results_backend.username = "default" + CONFIG.results_backend.cert_reqs = "none" + CONFIG.results_backend.encryption_key = key_file + CONFIG.results_backend.db_num = 0 + + # Go run the tests + yield + + # Reset the configuration + CONFIG.celery = orig_config.celery + CONFIG.broker = orig_config.broker + CONFIG.results_backend = orig_config.results_backend + + +@pytest.fixture(scope="function") +def redis_broker_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a Redis broker and results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + + CONFIG.broker.password = pass_file + CONFIG.broker.port = 6379 + CONFIG.broker.name = "redis" + + yield + + +@pytest.fixture(scope="function") +def redis_results_backend_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a Redis results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.port = 6379 + CONFIG.results_backend.name = "redis" + + yield + + +@pytest.fixture(scope="function") +def rabbit_broker_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a RabbitMQ broker. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/rabbit.pass" + create_pass_file(pass_file) + + CONFIG.broker.password = pass_file + CONFIG.broker.port = 5671 + CONFIG.broker.name = "rabbitmq" + + yield + + +@pytest.fixture(scope="function") +def mysql_results_backend_config( + merlin_server_dir: str, config: "fixture" # noqa: F821 pylint: disable=redefined-outer-name,unused-argument +): + """ + This fixture is intended to be used for testing any functionality in the codebase + that uses the CONFIG object with a MySQL results_backend. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param config: The fixture that sets up most of the CONFIG object for testing + """ + pass_file = f"{merlin_server_dir}/mysql.pass" + create_pass_file(pass_file) + + create_cert_files(merlin_server_dir, CERT_FILES) + + CONFIG.results_backend.password = pass_file + CONFIG.results_backend.name = "mysql" + CONFIG.results_backend.dbname = "test_mysql_db" + CONFIG.results_backend.keyfile = CERT_FILES["ssl_key"] + CONFIG.results_backend.certfile = CERT_FILES["ssl_cert"] + CONFIG.results_backend.ca_certs = CERT_FILES["ssl_ca"] + + yield diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 000000000..26cfe4c0a --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,11 @@ +""" +This module will store constants that will be used throughout our test suite. +""" + +SERVER_PASS = "merlin-test-server" + +CERT_FILES = { + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem", +} diff --git a/tests/context_managers/__init__.py b/tests/context_managers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/celery_test_workers.py b/tests/context_managers/celery_workers_manager.py similarity index 98% rename from tests/celery_test_workers.py rename to tests/context_managers/celery_workers_manager.py index ad81d30e6..5e750b6ef 100644 --- a/tests/celery_test_workers.py +++ b/tests/context_managers/celery_workers_manager.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # @@ -42,7 +42,7 @@ from celery import Celery -class CeleryTestWorkersManager: +class CeleryWorkersManager: """ A class to handle the setup and teardown of celery workers. This should be treated as a context and used with python's @@ -135,7 +135,7 @@ def start_worker(self, worker_launch_cmd: List[str]): app.worker_main instead of the normal "celery -A worker" command to launch the workers since our celery app is created in a pytest fixture and is unrecognizable by the celery command. For each worker, the output of it's logs are sent to - /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ under a file with a name + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/python_{major}.{minor}.{micro}_current/ under a file with a name similar to: test_worker_*.log. NOTE: pytest-current/ will have the results of the most recent test run. If you want to see a previous run check under pytest-/. HOWEVER, only the 3 most recent test runs will be saved. diff --git a/tests/context_managers/server_manager.py b/tests/context_managers/server_manager.py new file mode 100644 index 000000000..b99afb2c6 --- /dev/null +++ b/tests/context_managers/server_manager.py @@ -0,0 +1,106 @@ +""" +Module to define functionality for managing the containerized +server used for testing. +""" + +import os +import signal +import subprocess +from types import TracebackType +from typing import Type + +import redis +import yaml + + +class RedisServerError(Exception): + """ + Exception to signal that the server wasn't pinged properly. + """ + + +class ServerInitError(Exception): + """ + Exception to signal that there was an error initializing the server. + """ + + +class RedisServerManager: + """ + A class to handle the setup and teardown of a containerized redis server. + This should be treated as a context and used with python's built-in 'with' + statement. If you use it without this statement, beware that the processes + spun up here may never be stopped. + """ + + def __init__(self, server_dir: str, redis_pass: str): + self._redis_pass = redis_pass + self.server_dir = server_dir + self.host = "localhost" + self.port = 6379 + self.database = 0 + self.username = "default" + self.redis_server_uri = f"redis://{self.username}:{self._redis_pass}@{self.host}:{self.port}/{self.database}" + + def __enter__(self): + """This magic method is necessary for allowing this class to be used as a context manager""" + return self + + def __exit__(self, exc_type: Type[Exception], exc_value: Exception, traceback: TracebackType): + """ + This will always run at the end of a context with statement, even if an error is raised. + It's a safe way to ensure all of our server gets stopped no matter what. + """ + self.stop_server() + + def initialize_server(self): + """ + Initialize the setup for the local redis server. We'll write the folder to: + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/python_{major}.{minor}.{micro}_current/ + We'll set the password to be 'merlin-test-server' so it'll be easy to shutdown if necessary + """ + subprocess.run( + f"merlin server init; merlin server config -pwd {self._redis_pass}", shell=True, capture_output=True, text=True + ) + + # Check that the merlin server was initialized properly + if not os.path.exists(self.server_dir): + raise ServerInitError("The merlin server was not initialized properly.") + + def start_server(self): + """Attempt to start the local redis server.""" + try: + # Need to set LC_ALL='C' before starting the server or else redis causes a failure + subprocess.run("export LC_ALL='C'; merlin server start", shell=True, timeout=5) + except subprocess.TimeoutExpired: + pass + + # Ensure the server started properly + redis_client = redis.Redis( + host=self.host, port=self.port, db=self.database, password=self._redis_pass, username=self.username + ) + if not redis_client.ping(): + raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") + + def stop_server(self): + """Stop the server.""" + # Attempt to stop the server gracefully with `merlin server` + kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) + + # Check that the server was terminated + if "Merlin server terminated." not in kill_process.stderr: + # If it wasn't, try to kill the process by using the pid stored in a file created by `merlin server` + try: + with open(f"{self.server_dir}/merlin_server.pf", "r") as process_file: + server_process_info = yaml.load(process_file, yaml.Loader) + os.kill(int(server_process_info["image_pid"]), signal.SIGKILL) + # If the file can't be found then let's make sure there's even a redis-server process running + except FileNotFoundError as exc: + process_query = subprocess.run("ps ux", shell=True, text=True, capture_output=True) + # If there is a file running we didn't start it in this test run so we can't kill it + if "redis-server" in process_query.stdout: + raise RedisServerError( + "Found an active redis server but cannot stop it since there is no process file (merlin_server.pf). " + "Did you start this server before running tests?" + ) from exc + # No else here. If there's no redis-server process found then there's nothing to stop diff --git a/tests/fixtures/examples.py b/tests/fixtures/examples.py new file mode 100644 index 000000000..7c4626e3e --- /dev/null +++ b/tests/fixtures/examples.py @@ -0,0 +1,22 @@ +""" +Fixtures specifically for help testing the modules in the examples/ directory. +""" + +import os + +import pytest + + +@pytest.fixture(scope="session") +def examples_testing_dir(temp_output_dir: str) -> str: + """ + Fixture to create a temporary output directory for tests related to the examples functionality. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for examples tests + """ + testing_dir = f"{temp_output_dir}/examples_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir diff --git a/tests/fixtures/server.py b/tests/fixtures/server.py new file mode 100644 index 000000000..4f4a07a2c --- /dev/null +++ b/tests/fixtures/server.py @@ -0,0 +1,308 @@ +""" +Fixtures specifically for help testing the modules in the server/ directory. +""" + +import os +from argparse import Namespace +from typing import Dict, Union + +import pytest +import yaml + + +# pylint: disable=redefined-outer-name + + +@pytest.fixture(scope="session") +def server_testing_dir(temp_output_dir: str) -> str: + """ + Fixture to create a temporary output directory for tests related to the server functionality. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :returns: The path to the temporary testing directory for server tests + """ + testing_dir = f"{temp_output_dir}/server_testing" + if not os.path.exists(testing_dir): + os.mkdir(testing_dir) + + return testing_dir + + +@pytest.fixture(scope="session") +def server_redis_conf_file(server_testing_dir: str) -> str: + """ + Fixture to write a redis.conf file to the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :returns: The path to the redis configuration file we'll use for testing + """ + redis_conf_file = f"{server_testing_dir}/redis.conf" + file_contents = """ + # ip address + bind 127.0.0.1 + + # port + port 6379 + + # password + requirepass merlin_password + + # directory + dir ./ + + # snapshot + save 300 100 + + # db file + dbfilename dump.rdb + + # append mode + appendfsync everysec + + # append file + appendfilename appendonly.aof + + # dummy trailing comment + """.strip().replace( + " ", "" + ) + + with open(redis_conf_file, "w") as rcf: + rcf.write(file_contents) + + return redis_conf_file + + +@pytest.fixture(scope="session") +def server_redis_pass_file(server_testing_dir: str) -> str: + """ + Fixture to create a redis password file in the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :returns: The path to the redis password file + """ + redis_pass_file = f"{server_testing_dir}/redis.pass" + + with open(redis_pass_file, "w") as rpf: + rpf.write("server-tests-password") + + return redis_pass_file + + +@pytest.fixture(scope="session") +def server_users() -> Dict[str, Dict[str, str]]: + """ + Create a dictionary of two test users with identical configuration settings. + + :returns: A dict containing the two test users and their settings + """ + users = { + "default": { + "channels": "*", + "commands": "@all", + "hash_password": "1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a", + "keys": "*", + "status": "on", + }, + "test_user": { + "channels": "*", + "commands": "@all", + "hash_password": "1ba9249af0c73dacb0f9a70567126624076b5bee40de811e65f57eabcdaf490a", + "keys": "*", + "status": "on", + }, + } + return users + + +@pytest.fixture(scope="session") +def server_redis_users_file(server_testing_dir: str, server_users: dict) -> str: + """ + Fixture to write a redis.users file to the temporary output directory. + + If a test will modify this file with a file write, you should make a copy of + this file to modify instead. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :param server_users: A dict of test user configurations + :returns: The path to the redis user configuration file we'll use for testing + """ + redis_users_file = f"{server_testing_dir}/redis.users" + + with open(redis_users_file, "w") as ruf: + yaml.dump(server_users, ruf) + + return redis_users_file + + +@pytest.fixture(scope="class") +def server_container_config_data( + server_testing_dir: str, + server_redis_conf_file: str, + server_redis_pass_file: str, + server_redis_users_file: str, +) -> Dict[str, str]: + """ + Fixture to provide sample data for ContainerConfig tests. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :param server_redis_conf_file: A pytest fixture that defines a path to a redis configuration file + :param server_redis_pass_file: A pytest fixture that defines a path to a redis password file + :param server_redis_users_file: A pytest fixture that defines a path to a redis users file + :returns: A dict containing the necessary key/values for the ContainerConfig object + """ + + return { + "format": "singularity", + "image_type": "redis", + "image": "redis_latest.sif", + "url": "docker://redis", + "config": server_redis_conf_file.split("/")[-1], + "config_dir": server_testing_dir, + "pfile": "merlin_server.pf", + "pass_file": server_redis_pass_file.split("/")[-1], + "user_file": server_redis_users_file.split("/")[-1], + } + + +@pytest.fixture(scope="class") +def server_container_format_config_data() -> Dict[str, str]: + """ + Fixture to provide sample data for ContainerFormatConfig tests + + :returns: A dict containing the necessary key/values for the ContainerFormatConfig object + """ + return { + "command": "singularity", + "run_command": "{command} run -H {home_dir} {image} {config}", + "stop_command": "kill", + "pull_command": "{command} pull {image} {url}", + } + + +@pytest.fixture(scope="class") +def server_process_config_data() -> Dict[str, str]: + """ + Fixture to provide sample data for ProcessConfig tests + + :returns: A dict containing the necessary key/values for the ProcessConfig object + """ + return { + "status": "pgrep -P {pid}", + "kill": "kill {pid}", + } + + +@pytest.fixture(scope="class") +def server_server_config( + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], + server_container_format_config_data: Dict[str, str], +) -> Dict[str, Dict[str, str]]: + """ + Fixture to provide sample data for ServerConfig tests + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :returns: A dictionary containing each of the configuration dicts we'll need + """ + return { + "container": server_container_config_data, + "process": server_process_config_data, + "singularity": server_container_format_config_data, + } + + +@pytest.fixture(scope="function") +def server_app_yaml_contents( + server_redis_pass_file: str, + server_container_config_data: Dict[str, str], + server_process_config_data: Dict[str, str], +) -> Dict[str, Union[str, int]]: + """ + Fixture to create the contents of an app.yaml file. + + :param server_redis_pass_file: A pytest fixture that defines a path to a redis password file + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :returns: A dict with typical app.yaml contents + """ + contents = { + "broker": { + "cert_reqs": "none", + "name": "redis", + "password": server_redis_pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "testhost", + }, + "container": server_container_config_data, + "process": server_process_config_data, + "results_backend": { + "cert_reqs": "none", + "db_num": 0, + "name": "redis", + "password": server_redis_pass_file, + "port": 6379, + "server": "127.0.0.1", + "username": "default", + }, + } + return contents + + +@pytest.fixture(scope="function") +def server_app_yaml(server_testing_dir: str, server_app_yaml_contents: dict) -> str: + """ + Fixture to create an app.yaml file in the temporary output directory. + + NOTE this must be function scoped since server_app_yaml_contents is function scoped. + + :param server_testing_dir: A pytest fixture that defines a path to the output directory we'll write to + :param server_app_yaml_contents: A pytest fixture that creates a dict of contents for an app.yaml file + :returns: The path to the app.yaml file + """ + app_yaml_file = f"{server_testing_dir}/app.yaml" + + if not os.path.exists(app_yaml_file): + with open(app_yaml_file, "w") as ayf: + yaml.dump(server_app_yaml_contents, ayf) + + return app_yaml_file + + +@pytest.fixture(scope="function") +def server_process_file_contents() -> str: + """Fixture to represent process file contents.""" + return {"parent_pid": 123, "image_pid": 456, "port": 6379, "hostname": "dummy_server"} + + +@pytest.fixture(scope="function") +def server_config_server_args() -> Namespace: + """ + Setup an argparse Namespace with all args that the `config_server` + function will need. These can be modified on a test-by-test basis. + + :returns: An argparse Namespace with args needed by `config_server` + """ + return Namespace( + ipaddress=None, + port=None, + password=None, + directory=None, + snapshot_seconds=None, + snapshot_changes=None, + snapshot_file=None, + append_mode=None, + append_file=None, + add_user=None, + remove_user=None, + ) diff --git a/tests/fixtures/status.py b/tests/fixtures/status.py index f26cea37c..39a36f9bf 100644 --- a/tests/fixtures/status.py +++ b/tests/fixtures/status.py @@ -14,6 +14,9 @@ from tests.unit.study.status_test_files import status_test_variables +# pylint: disable=redefined-outer-name + + @pytest.fixture(scope="session") def status_testing_dir(temp_output_dir: str) -> str: """ @@ -29,8 +32,8 @@ def status_testing_dir(temp_output_dir: str) -> str: return testing_dir -@pytest.fixture(scope="session") -def status_empty_file(status_testing_dir: str) -> str: # pylint: disable=W0621 +@pytest.fixture(scope="class") +def status_empty_file(status_testing_dir: str) -> str: """ A pytest fixture to create an empty status file. diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 83f07aafe..1c852b064 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/definitions.py b/tests/integration/definitions.py index 59c1fa256..18435d461 100644 --- a/tests/integration/definitions.py +++ b/tests/integration/definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index ef2bef882..868ec1c3a 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/study/test_celeryadapter.py b/tests/integration/test_celeryadapter.py similarity index 99% rename from tests/unit/study/test_celeryadapter.py rename to tests/integration/test_celeryadapter.py index 0572d6c66..a40f1054b 100644 --- a/tests/unit/study/test_celeryadapter.py +++ b/tests/integration/test_celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/common/test_dumper.py b/tests/unit/common/test_dumper.py new file mode 100644 index 000000000..c52e9fe90 --- /dev/null +++ b/tests/unit/common/test_dumper.py @@ -0,0 +1,168 @@ +""" +Tests for the `dumper.py` file. +""" + +import csv +import json +import os +from datetime import datetime +from time import sleep + +import pytest + +from merlin.common.dumper import dump_handler + + +NUM_ROWS = 5 +CSV_INFO_TO_DUMP = { + "row_num": [i for i in range(1, NUM_ROWS + 1)], + "other_info": [f"test_info_{i}" for i in range(1, NUM_ROWS + 1)], +} +JSON_INFO_TO_DUMP = {str(i): {f"other_info_{i}": f"test_info_{i}"} for i in range(1, NUM_ROWS + 1)} +DUMP_HANDLER_DIR = "{temp_output_dir}/dump_handler" + + +def test_dump_handler_invalid_dump_file(): + """ + This is really testing the initialization of the Dumper class with an invalid file type. + This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + dump_handler("bad_file.txt", CSV_INFO_TO_DUMP) + assert "Invalid file type for bad_file.txt. Supported file types are: ['csv', 'json']" in str(excinfo.value) + + +def get_output_file(temp_dir: str, file_name: str): + """ + Helper function to get a full path to the temporary output file. + + :param temp_dir: The path to the temporary output directory that pytest gives us + :param file_name: The name of the file + """ + dump_dir = DUMP_HANDLER_DIR.format(temp_output_dir=temp_dir) + if not os.path.exists(dump_dir): + os.mkdir(dump_dir) + dump_file = f"{dump_dir}/{file_name}" + return dump_file + + +def run_csv_dump_test(dump_file: str, fmode: str): + """ + Run the test for csv dump. + + :param dump_file: The file that the dump was written to + :param fmode: The type of write that we're testing ("w" for write, "a" for append) + """ + + # Check that the file exists and that read in the contents of the file + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + reader = csv.reader(df) + written_data = list(reader) + + expected_rows = NUM_ROWS * 2 if fmode == "a" else NUM_ROWS + assert len(written_data) == expected_rows + 1 # Adding one because of the header row + for i, row in enumerate(written_data): + assert len(row) == 2 # Check number of columns + if i == 0: # Checking the header row + assert row[0] == "row_num" + assert row[1] == "other_info" + else: # Checking the data rows + assert row[0] == str(CSV_INFO_TO_DUMP["row_num"][(i % NUM_ROWS) - 1]) + assert row[1] == str(CSV_INFO_TO_DUMP["other_info"][(i % NUM_ROWS) - 1]) + + +def test_dump_handler_csv_write(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class. + This should create a csv file and write to it. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "csv_write.csv") + + # Run the actual call to dump to the file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Assert that everything ran properly + run_csv_dump_test(dump_file, "w") + + +def test_dump_handler_csv_append(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class with the file write mode set to append. + We'll write to a csv file first and then run again to make sure we can append to it properly. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "csv_append.csv") + + # Run the first call to create the csv file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Run the second call to append to the csv file + dump_handler(dump_file, CSV_INFO_TO_DUMP) + + # Assert that everything ran properly + run_csv_dump_test(dump_file, "a") + + +def test_dump_handler_json_write(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class. + This should create a json file and write to it. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "json_write.json") + + # Run the actual call to dump to the file + dump_handler(dump_file, JSON_INFO_TO_DUMP) + + # Check that the file exists and that the contents are correct + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + contents = json.load(df) + assert contents == JSON_INFO_TO_DUMP + + +def test_dump_handler_json_append(temp_output_dir: str): + """ + This is really testing the write method of the Dumper class with the file write mode set to append. + We'll write to a json file first and then run again to make sure we can append to it properly. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the path to the file we'll write to + dump_file = get_output_file(temp_output_dir, "json_append.json") + + # Run the first call to create the file + timestamp_1 = str(datetime.now()) + first_dump = {timestamp_1: JSON_INFO_TO_DUMP} + dump_handler(dump_file, first_dump) + + # Sleep so we don't accidentally get the same timestamp + sleep(0.5) + + # Run the second call to append to the file + timestamp_2 = str(datetime.now()) + second_dump = {timestamp_2: JSON_INFO_TO_DUMP} + dump_handler(dump_file, second_dump) + + # Check that the file exists and that the contents are correct + assert os.path.exists(dump_file) + with open(dump_file, "r") as df: + contents = json.load(df) + keys = contents.keys() + assert len(keys) == 2 + assert timestamp_1 in keys + assert timestamp_2 in keys + assert contents[timestamp_1] == JSON_INFO_TO_DUMP + assert contents[timestamp_2] == JSON_INFO_TO_DUMP diff --git a/tests/unit/common/test_encryption.py b/tests/unit/common/test_encryption.py new file mode 100644 index 000000000..3e37cef84 --- /dev/null +++ b/tests/unit/common/test_encryption.py @@ -0,0 +1,136 @@ +""" +Tests for the `encrypt.py` and `encrypt_backend_traffic.py` files. +""" + +import os + +import celery +import pytest + +from merlin.common.security.encrypt import _gen_key, _get_key, _get_key_path, decrypt, encrypt +from merlin.common.security.encrypt_backend_traffic import _decrypt_decode, _encrypt_encode, set_backend_funcs +from merlin.config.configfile import CONFIG + + +class TestEncryption: + """ + This class will house all tests necessary for our encryption modules. + """ + + def test_encrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test that our encryption function is encrypting the bytes that we're + passing to it. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + str_to_encrypt = b"super secret string shhh" + encrypted_str = encrypt(str_to_encrypt) + for word in str_to_encrypt.decode("utf-8").split(" "): + assert word not in encrypted_str.decode("utf-8") + + def test_decrypt(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test that our decryption function is decrypting the bytes that we're passing to it. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # This is the output of the bytes from the encrypt test + str_to_decrypt = b"gAAAAABld6k-jEncgCW5AePgrwn-C30dhr7dzGVhqzcqskPqFyA2Hdg3VWmo0qQnLklccaUYzAGlB4PMxyp4T-1gAYlAOf_7sC_bJOEcYOIkhZFoH6cX4Uw=" + decrypted_str = decrypt(str_to_decrypt) + assert decrypted_str == b"super secret string shhh" + + def test_get_key_path(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `_get_key_path` function. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Test the default behavior (`_get_key_path` will pull from CONFIG.results_backend which + # will be set to the temporary output path for our tests in the `use_fake_encrypt_data_key` fixture) + actual_default = _get_key_path() + assert actual_default.startswith("/tmp/") and actual_default.endswith("/encrypt_data_key") + + # Test with having the encryption key set to None + temp = CONFIG.results_backend.encryption_key + CONFIG.results_backend.encryption_key = None + with pytest.raises(ValueError) as excinfo: + _get_key_path() + assert "Error! No password provided for RabbitMQ" in str(excinfo.value) + CONFIG.results_backend.encryption_key = temp + + # Test with having the entire results_backend wiped from CONFIG + orig_results_backend = CONFIG.results_backend + CONFIG.results_backend = None + actual_no_results_backend = _get_key_path() + assert actual_no_results_backend == os.path.abspath(os.path.expanduser("~/.merlin/encrypt_data_key")) + CONFIG.results_backend = orig_results_backend + + def test_gen_key(self, temp_output_dir: str): + """ + Test the `_gen_key` function. + + :param temp_output_dir: The path to the temporary output directory for this test run + """ + # Create the file but don't put anything in it + key_gen_test_file = f"{temp_output_dir}/key_gen_test" + with open(key_gen_test_file, "w"): + pass + + # Ensure nothing is in the file + with open(key_gen_test_file, "r") as key_gen_file: + key_gen_contents = key_gen_file.read() + assert key_gen_contents == "" + + # Run the test and then check to make sure the file is now populated + _gen_key(key_gen_test_file) + with open(key_gen_test_file, "r") as key_gen_file: + key_gen_contents = key_gen_file.read() + assert key_gen_contents != "" + + def test_get_key( + self, merlin_server_dir: str, test_encryption_key: bytes, redis_results_backend_config: "fixture" # noqa: F821 + ): + """ + Test the `_get_key` function. + + :param merlin_server_dir: The directory to the merlin test server configuration + :param test_encryption_key: A fixture to establish a fixed encryption key for testing + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Test the default functionality + actual_default = _get_key() + assert actual_default == test_encryption_key + + # Modify the permission of the key file so that it can't be read by anyone + # (we're purposefully trying to raise an IOError) + key_path = f"{merlin_server_dir}/encrypt_data_key" + orig_file_permissions = os.stat(key_path).st_mode + os.chmod(key_path, 0o222) + with pytest.raises(IOError): + _get_key() + os.chmod(key_path, orig_file_permissions) + + # Reset the key value to our test value since the IOError test will rewrite the key + with open(key_path, "w") as key_file: + key_file.write(test_encryption_key.decode("utf-8")) + + def test_set_backend_funcs(self): + """ + Test the `set_backend_funcs` function. + """ + orig_encode = celery.backends.base.Backend.encode + orig_decode = celery.backends.base.Backend.decode + + # Make sure these values haven't been set yet + assert celery.backends.base.Backend.encode != _encrypt_encode + assert celery.backends.base.Backend.decode != _decrypt_decode + + set_backend_funcs() + + # Ensure the new functions have been set + assert celery.backends.base.Backend.encode == _encrypt_encode + assert celery.backends.base.Backend.decode == _decrypt_decode + + celery.backends.base.Backend.encode = orig_encode + celery.backends.base.Backend.decode = orig_decode diff --git a/tests/unit/common/test_sample_index.py b/tests/unit/common/test_sample_index.py index c693827f0..c9cd108ee 100644 --- a/tests/unit/common/test_sample_index.py +++ b/tests/unit/common/test_sample_index.py @@ -1,178 +1,519 @@ +""" +Tests for the `sample_index.py` and `sample_index_factory.py` files. +""" + import os -import shutil -from contextlib import suppress +import pytest + +from merlin.common.sample_index import SampleIndex, new_dir, uniform_directories from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy -TEST_DIR = "UNIT_TEST_SPACE" +def test_uniform_directories(): + """ + Test the `uniform_directories` function with different inputs. + """ + # Create the tests and the expected outputs + tests = [ + # SMALL SAMPLE SIZE + (10, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (10, 1, 2), + (10, 2, 100), + (10, 2, 2), + # MEDIUM SAMPLE SIZE + (10000, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (10000, 1, 5), + (10000, 5, 100), + (10000, 5, 10), + # LARGE SAMPLE SIZE + (1000000000, 1, 100), # Bundle size of 1 and max dir level of 100 is default + (1000000000, 1, 5), + (1000000000, 5, 100), + (1000000000, 5, 10), + ] + expected_outputs = [ + # SMALL SAMPLE SIZE + [1], + [8, 4, 2, 1], + [2], + [8, 4, 2], + # MEDIUM SAMPLE SIZE + [100, 1], + [3125, 625, 125, 25, 5, 1], + [500, 5], + [5000, 500, 50, 5], + # LARGE SAMPLE SIZE + [100000000, 1000000, 10000, 100, 1], + [244140625, 48828125, 9765625, 1953125, 390625, 78125, 15625, 3125, 625, 125, 25, 5, 1], + [500000000, 5000000, 50000, 500, 5], + [500000000, 50000000, 5000000, 500000, 50000, 5000, 500, 50, 5], + ] + # Run the tests and compare outputs + for i, test in enumerate(tests): + actual = uniform_directories(num_samples=test[0], bundle_size=test[1], level_max_dirs=test[2]) + assert actual == expected_outputs[i] -def clear_test_tree(): - with suppress(FileNotFoundError): - shutil.rmtree(TEST_DIR) +def test_new_dir(temp_output_dir: str): + """ + Test the `new_dir` function. This will test a valid path and also raising an OSError during + creation. -def clear(func): - def wrapper(): - clear_test_tree() - func() - clear_test_tree() + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Test basic functionality + test_path = f"{os.getcwd()}/test_sample_index/test_new_dir" + new_dir(test_path) + assert os.path.exists(test_path) - return wrapper + # Test OSError functionality + new_dir(test_path) -@clear -def test_index_file_writing(): - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) - indx.write_directories() - indx.write_multiple_sample_index_files() - indx2 = read_hierarchy(TEST_DIR) - assert indx2.get_path_to_sample(123000123) == indx.get_path_to_sample(123000123) +class TestSampleIndex: + """ + These tests focus on testing the SampleIndex class used for creating the + sample hierarchy. + NOTE to see output of creating any hierarchy, change `write_all_hierarchies` to True. + The results of each hierarchy will be written to: + /tmp/`whoami`/pytest/pytest-of-`whoami`/pytest-current/python_{major}.{minor}.{micro}_current/test_sample_index/ + """ -def test_bundle_retrieval(): - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=TEST_DIR) - expected = f"{TEST_DIR}/0/0/0/samples0-10000.ext" - result = indx.get_path_to_sample(123) - assert expected == result + write_all_hierarchies = False - expected = f"{TEST_DIR}/0/0/0/samples10000-20000.ext" - result = indx.get_path_to_sample(10000) - assert expected == result + def get_working_dir(self, test_workspace: str): + """ + This method is called for every test to get a unique workspace in the temporary + directory for the test output. - expected = f"{TEST_DIR}/1/2/3/samples123000000-123010000.ext" - result = indx.get_path_to_sample(123000123) - assert expected == result + :param test_workspace: The unique name for this workspace + (all tests use their unique test name for this value usually) + """ + return f"{os.getcwd()}/test_sample_index/{test_workspace}" + def write_hierarchy_for_debug(self, indx: SampleIndex): + """ + This method is for debugging purposes. It will cause all tests that don't write + hierarchies to write them so the output can be investigated. -def test_start_sample_id(): - expected = """: DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10 - 0: BUNDLE 0 MIN 203 MAX 213 - 1: BUNDLE 1 MIN 213 MAX 223 - 2: BUNDLE 2 MIN 223 MAX 233 - 3: BUNDLE 3 MIN 233 MAX 243 - 4: BUNDLE 4 MIN 243 MAX 253 - 5: BUNDLE 5 MIN 253 MAX 263 - 6: BUNDLE 6 MIN 263 MAX 273 - 7: BUNDLE 7 MIN 273 MAX 283 - 8: BUNDLE 8 MIN 283 MAX 293 - 9: BUNDLE 9 MIN 293 MAX 303 -""" - idx203 = create_hierarchy(100, 10, start_sample_id=203) - assert expected == str(idx203) - - -@clear -def test_directory_writing(): - path = os.path.join(TEST_DIR) - indx = create_hierarchy(2, 1, [1], root=path) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE 0 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: BUNDLE 1 MIN 1 MAX 2 -""" - assert expected == str(indx) - indx.write_directories() - assert os.path.isdir(f"{TEST_DIR}/0") - assert os.path.isdir(f"{TEST_DIR}/1") - indx.write_multiple_sample_index_files() - - clear_test_tree() - - path = os.path.join(TEST_DIR) - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000], root=path) - indx.write_directories() - path = indx.get_path_to_sample(123000123) - assert os.path.exists(os.path.dirname(path)) - assert path != TEST_DIR - path = indx.get_path_to_sample(10000000000) - assert path == TEST_DIR - - clear_test_tree() - - path = os.path.join(TEST_DIR) - indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=path) - indx.write_directories() - - -def test_directory_path(): - indx = create_hierarchy(20, 1, [20, 5, 1], root="") - leaves = indx.make_directory_string() - expected_leaves = "0/0/0 0/0/1 0/0/2 0/0/3 0/0/4 0/1/0 0/1/1 0/1/2 0/1/3 0/1/4 0/2/0 0/2/1 0/2/2 0/2/3 0/2/4 0/3/0 0/3/1 0/3/2 0/3/3 0/3/4" - assert leaves == expected_leaves - all_dirs = indx.make_directory_string(just_leaf_directories=False) - expected_all_dirs = " 0 0/0 0/0/0 0/0/1 0/0/2 0/0/3 0/0/4 0/1 0/1/0 0/1/1 0/1/2 0/1/3 0/1/4 0/2 0/2/0 0/2/1 0/2/2 0/2/3 0/2/4 0/3 0/3/0 0/3/1 0/3/2 0/3/3 0/3/4" - assert all_dirs == expected_all_dirs - - -@clear -def test_subhierarchy_insertion(): - indx = create_hierarchy(2, 1, [1], root=TEST_DIR) - print("Writing directories") - indx.write_directories() - indx.write_multiple_sample_index_files() - print("reading heirarchy") - top = read_hierarchy(os.path.abspath(TEST_DIR)) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE -1 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: BUNDLE -1 MIN 1 MAX 2 -""" - assert str(top) == expected - print("creating sub_heirarchy") - sub_h = create_hierarchy(100, 10, address="1.0") - print("inserting sub_heirarchy") - top["1.0"] = sub_h - print(str(indx)) - print("after insertion") - print(str(top)) - expected = """: DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2 - 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1 - 0.0: BUNDLE -1 MIN 0 MAX 1 - 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1 - 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10 - 1.0.0: BUNDLE 0 MIN 0 MAX 10 - 1.0.1: BUNDLE 1 MIN 10 MAX 20 - 1.0.2: BUNDLE 2 MIN 20 MAX 30 - 1.0.3: BUNDLE 3 MIN 30 MAX 40 - 1.0.4: BUNDLE 4 MIN 40 MAX 50 - 1.0.5: BUNDLE 5 MIN 50 MAX 60 - 1.0.6: BUNDLE 6 MIN 60 MAX 70 - 1.0.7: BUNDLE 7 MIN 70 MAX 80 - 1.0.8: BUNDLE 8 MIN 80 MAX 90 - 1.0.9: BUNDLE 9 MIN 90 MAX 100 -""" - assert str(top) == expected + :param indx: The `SampleIndex` object to write the hierarchy for + """ + if self.write_all_hierarchies: + indx.write_directories() + indx.write_multiple_sample_index_files() + def test_invalid_children(self): + """ + This will test that an invalid type for the `children` argument will raise + an error. + """ + tests = [ + ["a", "b", "c"], + True, + "a b c", + ] + for test in tests: + with pytest.raises(TypeError): + SampleIndex(0, 10, test, "name") -def test_sample_index(): - """Run through some basic testing of the SampleIndex class.""" - tests = [ - (10, 1, []), - (10, 3, []), - (11, 2, [5]), - (10, 3, [3]), - (10, 3, [1]), - (10, 1, [3]), - (10, 3, [1, 3]), - (10, 1, [2]), - (1000, 100, [500]), - (1000, 50, [500, 100]), - (1000000000, 100000132, []), - ] + def test_is_parent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_parent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_parent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if parent of leaf is recognized + assert indx.is_parent_of_leaf is False + assert indx.children["0"].is_parent_of_leaf is True + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"] + assert leaf_node.is_parent_of_leaf is False + + def test_is_grandparent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_grandparent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if grandparent of leaf is recognized + assert indx.is_grandparent_of_leaf is True + assert indx.children["0"].is_grandparent_of_leaf is False + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"] + assert leaf_node.is_grandparent_of_leaf is False + + def test_is_great_grandparent_of_leaf(self, temp_output_dir: str): + """ + Test the `is_great_grandparent_of_leaf` property. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_great_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [5, 1], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test to see if great grandparent of leaf is recognized + assert indx.is_great_grandparent_of_leaf is True + assert indx.children["0"].is_great_grandparent_of_leaf is False + assert indx.children["0"].children["0.0"].is_great_grandparent_of_leaf is False + + # Test to see if leaf is recognized + leaf_node = indx.children["0"].children["0.0"].children["0.0.0"] + assert leaf_node.is_great_grandparent_of_leaf is False + + def test_traverse_bundle(self, temp_output_dir: str): + """ + Test the `traverse_bundle` method to make sure it's just returning leaves. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Ensure all nodes in the traversal are leaves + for _, node in indx.traverse_bundles(): + assert node.is_leaf + + def test_getitem(self, temp_output_dir: str): + """ + Test the `__getitem__` magic method. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test getting that requesting the root returns itself + assert indx[""] == indx + + # Test a valid address + assert indx["0"] == indx.children["0"] + + # Test an invalid address + with pytest.raises(KeyError): + indx["10"] + + def test_setitem(self, temp_output_dir: str): + """ + Test the `__setitem__` magic method. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create a hierarchy to test + working_dir = self.get_working_dir("test_is_grandparent_of_leaf") + indx = create_hierarchy(10, 1, [2], root=working_dir) + self.write_hierarchy_for_debug(indx) + + invalid_indx = SampleIndex(1, 3, {}, "invalid_indx") + + # Ensure that trying to change the root raises an error + with pytest.raises(KeyError): + indx[""] = invalid_indx + + # Ensure we can't just add a new subtree to a level + with pytest.raises(KeyError): + indx["10"] = invalid_indx + + # Test that invalid subtrees are caught + with pytest.raises(TypeError): + indx["0"] = invalid_indx + + # Test a valid set operation + dummy_indx = SampleIndex(0, 1, {}, "dummy_indx", leafid=0, address="0.0") + indx["0"]["0.0"] = dummy_indx + + def test_index_file_writing(self, temp_output_dir: str): + """ + Test the functionality of writing multiple index files. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_index_file_writing") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + indx.write_directories() + indx.write_multiple_sample_index_files() + indx2 = read_hierarchy(working_dir) + assert indx2.get_path_to_sample(123000123) == indx.get_path_to_sample(123000123) + + def test_directory_writing_small(self, temp_output_dir: str): + """ + Test that writing a small directory functions properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the directory and ensure it has the correct format + working_dir = self.get_working_dir("test_directory_writing_small/") + indx = create_hierarchy(2, 1, [1], root=working_dir) + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE 0 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: BUNDLE 1 MIN 1 MAX 2\n" + ) + assert expected == str(indx) + + # Write the directories and ensure the paths are actually written + indx.write_directories() + assert os.path.isdir(f"{working_dir}/0") + assert os.path.isdir(f"{working_dir}/1") + indx.write_multiple_sample_index_files() + + def test_directory_writing_large(self, temp_output_dir: str): + """ + Test that writing a large directory functions properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_directory_writing_large") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + indx.write_directories() + path = indx.get_path_to_sample(123000123) + assert os.path.exists(os.path.dirname(path)) + assert path != working_dir + path = indx.get_path_to_sample(10000000000) + assert path == working_dir + + def test_bundle_retrieval(self, temp_output_dir: str): + """ + Test the functionality to get a bundle of samples when providing a sample id to find. + This will test a large sample hierarchy to ensure this scales properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the hierarchy + working_dir = self.get_working_dir("test_bundle_retrieval") + indx = create_hierarchy(1000000000, 10000, [100000000, 10000000, 1000000], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Test for a small sample id + expected = f"{working_dir}/0/0/0/samples0-10000.ext" + result = indx.get_path_to_sample(123) + assert expected == result + + # Test for a mid size sample id + expected = f"{working_dir}/0/0/0/samples10000-20000.ext" + result = indx.get_path_to_sample(10000) + assert expected == result + + # Test for a large sample id + expected = f"{working_dir}/1/2/3/samples123000000-123010000.ext" + result = indx.get_path_to_sample(123000123) + assert expected == result + + def test_start_sample_id(self, temp_output_dir: str): + """ + Test creating a hierarchy using a starting sample id. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + working_dir = self.get_working_dir("test_start_sample_id") + expected = ( + ": DIRECTORY MIN 203 MAX 303 NUM_BUNDLES 10\n" + " 0: BUNDLE 0 MIN 203 MAX 213\n" + " 1: BUNDLE 1 MIN 213 MAX 223\n" + " 2: BUNDLE 2 MIN 223 MAX 233\n" + " 3: BUNDLE 3 MIN 233 MAX 243\n" + " 4: BUNDLE 4 MIN 243 MAX 253\n" + " 5: BUNDLE 5 MIN 253 MAX 263\n" + " 6: BUNDLE 6 MIN 263 MAX 273\n" + " 7: BUNDLE 7 MIN 273 MAX 283\n" + " 8: BUNDLE 8 MIN 283 MAX 293\n" + " 9: BUNDLE 9 MIN 293 MAX 303\n" + ) + idx203 = create_hierarchy(100, 10, start_sample_id=203, root=working_dir) + self.write_hierarchy_for_debug(idx203) + + assert expected == str(idx203) + + def test_make_directory_string(self, temp_output_dir: str): + """ + Test the `make_directory_string` method of `SampleIndex`. This will check + both the normal functionality where we just request paths to the leaves and + also the inverse functionality where we request all paths that are not leaves. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Creating the hierarchy + working_dir = self.get_working_dir("test_make_directory_string") + indx = create_hierarchy(20, 1, [20, 5, 1], root=working_dir) + self.write_hierarchy_for_debug(indx) + + # Testing normal functionality (just leaf directories) + leaves = indx.make_directory_string() + expected_leaves_list = [ + f"{working_dir}/0/0/0", + f"{working_dir}/0/0/1", + f"{working_dir}/0/0/2", + f"{working_dir}/0/0/3", + f"{working_dir}/0/0/4", + f"{working_dir}/0/1/0", + f"{working_dir}/0/1/1", + f"{working_dir}/0/1/2", + f"{working_dir}/0/1/3", + f"{working_dir}/0/1/4", + f"{working_dir}/0/2/0", + f"{working_dir}/0/2/1", + f"{working_dir}/0/2/2", + f"{working_dir}/0/2/3", + f"{working_dir}/0/2/4", + f"{working_dir}/0/3/0", + f"{working_dir}/0/3/1", + f"{working_dir}/0/3/2", + f"{working_dir}/0/3/3", + f"{working_dir}/0/3/4", + ] + expected_leaves = " ".join(expected_leaves_list) + assert leaves == expected_leaves + + # Testing no leaf functionality + all_dirs = indx.make_directory_string(just_leaf_directories=False) + expected_all_dirs_list = [ + working_dir, + f"{working_dir}/0", + f"{working_dir}/0/0", + f"{working_dir}/0/0/0", + f"{working_dir}/0/0/1", + f"{working_dir}/0/0/2", + f"{working_dir}/0/0/3", + f"{working_dir}/0/0/4", + f"{working_dir}/0/1", + f"{working_dir}/0/1/0", + f"{working_dir}/0/1/1", + f"{working_dir}/0/1/2", + f"{working_dir}/0/1/3", + f"{working_dir}/0/1/4", + f"{working_dir}/0/2", + f"{working_dir}/0/2/0", + f"{working_dir}/0/2/1", + f"{working_dir}/0/2/2", + f"{working_dir}/0/2/3", + f"{working_dir}/0/2/4", + f"{working_dir}/0/3", + f"{working_dir}/0/3/0", + f"{working_dir}/0/3/1", + f"{working_dir}/0/3/2", + f"{working_dir}/0/3/3", + f"{working_dir}/0/3/4", + ] + expected_all_dirs = " ".join(expected_all_dirs_list) + assert all_dirs == expected_all_dirs + + def test_subhierarchy_insertion(self, temp_output_dir: str): + """ + Test that a subhierarchy can be inserted into our `SampleIndex` properly. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Create the hierarchy and read it + working_dir = self.get_working_dir("test_subhierarchy_insertion") + indx = create_hierarchy(2, 1, [1], root=working_dir) + indx.write_directories() + indx.write_multiple_sample_index_files() + top = read_hierarchy(os.path.abspath(working_dir)) + + # Compare results + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: BUNDLE -1 MIN 1 MAX 2\n" + ) + assert str(top) == expected + + # Create and insert the sub hierarchy + sub_h = create_hierarchy(100, 10, address="1.0") + top["1.0"] = sub_h + + # Compare results + expected = ( + ": DIRECTORY MIN 0 MAX 2 NUM_BUNDLES 2\n" + " 0: DIRECTORY MIN 0 MAX 1 NUM_BUNDLES 1\n" + " 0.0: BUNDLE -1 MIN 0 MAX 1\n" + " 1: DIRECTORY MIN 1 MAX 2 NUM_BUNDLES 1\n" + " 1.0: DIRECTORY MIN 0 MAX 100 NUM_BUNDLES 10\n" + " 1.0.0: BUNDLE 0 MIN 0 MAX 10\n" + " 1.0.1: BUNDLE 1 MIN 10 MAX 20\n" + " 1.0.2: BUNDLE 2 MIN 20 MAX 30\n" + " 1.0.3: BUNDLE 3 MIN 30 MAX 40\n" + " 1.0.4: BUNDLE 4 MIN 40 MAX 50\n" + " 1.0.5: BUNDLE 5 MIN 50 MAX 60\n" + " 1.0.6: BUNDLE 6 MIN 60 MAX 70\n" + " 1.0.7: BUNDLE 7 MIN 70 MAX 80\n" + " 1.0.8: BUNDLE 8 MIN 80 MAX 90\n" + " 1.0.9: BUNDLE 9 MIN 90 MAX 100\n" + ) + assert str(top) == expected + + def test_sample_index_creation_and_insertion(self, temp_output_dir: str): + """ + Run through some basic testing of the SampleIndex class. This will try + creating hierarchies of different sizes and inserting subhierarchies of + different sizes as well. + + :param temp_output_dir: A pytest fixture defined in conftest.py that creates a + temporary output path for our tests + """ + # Define the tests for hierarchies of varying sizes + tests = [ + (10, 1, []), + (10, 3, []), + (11, 2, [5]), + (10, 3, [3]), + (10, 3, [1]), + (10, 1, [3]), + (10, 3, [1, 3]), + (10, 1, [2]), + (1000, 100, [500]), + (1000, 50, [500, 100]), + (1000000000, 100000132, []), + ] + + # Run all the tests we defined above + for i, args in enumerate(tests): + working_dir = self.get_working_dir(f"test_sample_index_creation_and_insertion/{i}") + + # Put at root address of "0" to guarantee insertion at "0.1" later is valid + idx = create_hierarchy(args[0], args[1], args[2], address="0", root=working_dir) + self.write_hierarchy_for_debug(idx) - for args in tests: - print(f"############ TEST {args[0]} {args[1]} {args[2]} ###########") - # put at root address of "0" to guarantee insertion at "0.1" later is valid - idx = create_hierarchy(args[0], args[1], args[2], address="0") - print(str(idx)) - try: - idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") - print("successful set") - print(str(idx)) - except KeyError as error: - print(error) - assert False + # Inserting hierarchy at 0.1 + try: + idx["0.1"] = create_hierarchy(args[0], args[1], args[2], address="0.1") + except KeyError: + assert False diff --git a/tests/unit/common/test_util_sampling.py b/tests/unit/common/test_util_sampling.py new file mode 100644 index 000000000..b4cc252d5 --- /dev/null +++ b/tests/unit/common/test_util_sampling.py @@ -0,0 +1,45 @@ +""" +Tests for the `util_sampling.py` file. +""" + +import numpy as np +import pytest + +from merlin.common.util_sampling import scale_samples + + +class TestUtilSampling: + """ + This class will hold all of the tests for the `util_sampling.py` file. + """ + + def test_scale_samples_basic(self): + """Test basic functionality""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(-1, 1), (2, 6)] + result = scale_samples(samples_norm, limits) + expected_result = np.array([[-0.6, 3.6], [0.2, 5.2]]) + np.testing.assert_array_almost_equal(result, expected_result) + + def test_scale_samples_logarithmic(self): + """Test functionality with log enabled""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(1, 5), (1, 100)] + result = scale_samples(samples_norm, limits, do_log=[False, True]) + expected_result = np.array([[1.8, 6.309573], [3.4, 39.810717]]) + np.testing.assert_array_almost_equal(result, expected_result) + + def test_scale_samples_invalid_input(self): + """Test that function raises ValueError for invalid input""" + with pytest.raises(ValueError): + # Invalid input: samples_norm should be a 2D array + scale_samples([0.2, 0.4, 0.6], [(1, 5), (2, 6)]) + + def test_scale_samples_with_custom_limits_norm(self): + """Test functionality with custom limits_norm""" + samples_norm = np.array([[0.2, 0.4], [0.6, 0.8]]) + limits = [(1, 5), (2, 6)] + limits_norm = (-1, 1) + result = scale_samples(samples_norm, limits, limits_norm=limits_norm) + expected_result = np.array([[3.4, 4.8], [4.2, 5.6]]) + np.testing.assert_array_almost_equal(result, expected_result) diff --git a/tests/unit/config/dummy_app.yaml b/tests/unit/config/dummy_app.yaml new file mode 100644 index 000000000..966156566 --- /dev/null +++ b/tests/unit/config/dummy_app.yaml @@ -0,0 +1,33 @@ +broker: + cert_reqs: none + name: redis + password: redis.pass + port: '6379' + server: 127.0.0.1 + username: default + vhost: host4gunny +celery: + override: + visibility_timeout: 86400 +container: + config: redis.conf + config_dir: ./merlin_server/ + format: singularity + image: redis_latest.sif + image_type: redis + pass_file: redis.pass + pfile: merlin_server.pf + url: docker://redis + user_file: redis.users +process: + kill: kill {pid} + status: pgrep -P {pid} +results_backend: + cert_reqs: none + db_num: 0 + encryption_key: encrypt_data_key + name: redis + password: redis.pass + port: '6379' + server: 127.0.0.1 + username: default \ No newline at end of file diff --git a/tests/unit/config/old_test_configfile.py b/tests/unit/config/old_test_configfile.py deleted file mode 100644 index 39139ec11..000000000 --- a/tests/unit/config/old_test_configfile.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Tests for the configfile module.""" - -import os -import shutil -import tempfile -import unittest -from getpass import getuser - -from merlin.config import configfile - -from .utils import mkfile - - -CONFIG_FILE_CONTENTS = """ -celery: - certs: path/to/celery/config/files - -broker: - name: rabbitmq - username: testuser - password: rabbit.password # The filename that contains the password. - server: jackalope.llnl.gov - -results_backend: - name: mysql - dbname: testuser - username: mlsi - password: mysql.password # The filename that contains the password. - server: rabbit.llnl.gov - -""" - - -class TestFindConfigFile(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.appfile = mkfile(self.tmpdir, "app.yaml") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_tempdir(self): - self.assertTrue(os.path.isdir(self.tmpdir)) - - def test_find_config_file(self): - """ - Given the path to a vaild config file, find and return the full - filepath. - """ - path = configfile.find_config_file(path=self.tmpdir) - expected = os.path.join(self.tmpdir, self.appfile) - self.assertEqual(path, expected) - - def test_find_config_file_error(self): - """Given an invalid path, return None.""" - invalid = "invalid/path" - expected = None - - path = configfile.find_config_file(path=invalid) - self.assertEqual(path, expected) - - -class TestConfigFile(unittest.TestCase): - """Unit tests for loading the config file.""" - - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - self.configfile = mkfile(self.tmpdir, "app.yaml", content=CONFIG_FILE_CONTENTS) - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_get_config(self): - """ - Given the directory path to a valid merlin config file, then - `get_config` should find the merlin config file and load the YAML - contents to a dictionary. - """ - expected = { - "broker": { - "name": "rabbitmq", - "password": "rabbit.password", - "server": "jackalope.llnl.gov", - "username": "testuser", - "vhost": getuser(), - }, - "celery": {"certs": "path/to/celery/config/files"}, - "results_backend": { - "dbname": "testuser", - "name": "mysql", - "password": "mysql.password", - "server": "rabbit.llnl.gov", - "username": "mlsi", - }, - } - - self.assertDictEqual(configfile.get_config(self.tmpdir), expected) diff --git a/tests/unit/config/old_test_results_backend.py b/tests/unit/config/old_test_results_backend.py deleted file mode 100644 index 638f13eb8..000000000 --- a/tests/unit/config/old_test_results_backend.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Tests for the results_backend module.""" - -import os -import shutil -import tempfile -import unittest - -from merlin.config import results_backend - -from .utils import mkfile - - -class TestResultsBackend(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() - - # Create test files. - self.tmpfile1 = mkfile(self.tmpdir, "mysql_test1.txt") - self.tmpfile2 = mkfile(self.tmpdir, "mysql_test2.txt") - - def tearDown(self): - shutil.rmtree(self.tmpdir, ignore_errors=True) - - def test_mysql_config(self): - """ - Given the path to a directory containing the MySQL cert files and a - dictionary of files to look for, then find and return the full path to - all the certs. - """ - certs = {"test1": "mysql_test1.txt", "test2": "mysql_test2.txt"} - - # This will just be the above dictionary with the full file paths. - expected = { - "test1": os.path.join(self.tmpdir, certs["test1"]), - "test2": os.path.join(self.tmpdir, certs["test2"]), - } - results = results_backend.get_mysql_config(self.tmpdir, certs) - self.assertDictEqual(results, expected) - - def test_mysql_config_no_files(self): - """ - Given the path to a directory containing the MySQL cert files and - an empty dictionary, then `get_mysql_config` should return an empty - dictionary. - """ - files = {} - result = results_backend.get_mysql_config(self.tmpdir, files) - self.assertEqual(result, {}) - - -class TestConfingMysqlErrorPath(unittest.TestCase): - """ - Test `get_mysql_config` against cases were the given path does not exist. - """ - - def test_mysql_config_false(self): - """ - Given a path that does not exist, then `get_mysql_config` should return - False. - """ - path = "invalid/path" - - # We don't need the dictionary populated for this test. The function - # should return False before trying to process the dictionary. - certs = {} - result = results_backend.get_mysql_config(path, certs) - self.assertFalse(result) diff --git a/tests/unit/config/test_broker.py b/tests/unit/config/test_broker.py new file mode 100644 index 000000000..581b19488 --- /dev/null +++ b/tests/unit/config/test_broker.py @@ -0,0 +1,551 @@ +""" +Tests for the `broker.py` file. +""" + +import os +from ssl import CERT_NONE +from typing import Any, Dict + +import pytest + +from merlin.config.broker import ( + RABBITMQ_CONNECTION, + REDISSOCK_CONNECTION, + get_connection_string, + get_rabbit_connection, + get_redis_connection, + get_redissock_connection, + get_ssl_config, + read_file, +) +from merlin.config.configfile import CONFIG +from tests.constants import SERVER_PASS +from tests.utils import create_pass_file + + +def test_read_file(merlin_server_dir: str): + """ + Test the `read_file` function. We'll start up our containerized redis server + so that we have a password file to read here. + + :param merlin_server_dir: The directory to the merlin test server configuration + """ + pass_file = f"{merlin_server_dir}/redis.pass" + create_pass_file(pass_file) + actual = read_file(pass_file) + assert actual == SERVER_PASS + + +def test_get_connection_string_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with an invalid broker (a broker that isn't one of: + ["rabbitmq", "redis", "rediss", "redis+socket", "amqps", "amqp"]). + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "invalid_broker" + with pytest.raises(ValueError): + get_connection_string() + + +def test_get_connection_string_no_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function without a broker name value in the CONFIG object. This + should raise a ValueError just like the `test_get_connection_string_invalid_broker` does. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.name + with pytest.raises(ValueError): + get_connection_string() + + +def test_get_connection_string_simple(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function in the simplest way that we can. This function + will automatically check for a broker url and if it finds one in the CONFIG object it will just + return the value it finds. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + test_url = "test_url" + CONFIG.broker.url = test_url + actual = get_connection_string() + assert actual == test_url + + +def test_get_ssl_config_no_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function without a broker. This should return False. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.name + assert not get_ssl_config() + + +class TestRabbitBroker: + """ + This class will house all tests necessary for our broker module when using a + rabbit broker. + """ + + def run_get_rabbit_connection(self, expected_vals: Dict[str, Any], include_password: bool, conn: str): + """ + Helper method to run the tests for the `get_rabbit_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"conn": "", + "vhost": "host4testing", + "username": "default", + "password": "", + "server": "127.0.0.1", + "port": } + :param include_password: If True, include the password in the output. Otherwise don't. + :param conn: The connection type to pass in (either amqp or amqps) + """ + expected = RABBITMQ_CONNECTION.format(**expected_vals) + actual = get_rabbit_connection(include_password=include_password, conn=conn) + assert actual == expected + + def test_get_rabbit_connection(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_dont_include_password(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function but set include_password to False. This should * out the + password + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": "******", + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=False, conn=conn) + + def test_get_rabbit_connection_no_port_amqp(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use + 5672 as the port since we're using amqp as the connection. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + CONFIG.broker.name = "amqp" + conn = "amqp" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5672, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_no_port_amqps(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no port in the CONFIG object. This should use + 5671 as the port since we're using amqps as the connection. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + conn = "amqps" + expected_vals = { + "conn": conn, + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_rabbit_connection(expected_vals=expected_vals, include_password=True, conn=conn) + + def test_get_rabbit_connection_no_password(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with no password file set. This should raise a ValueError. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.password + with pytest.raises(ValueError) as excinfo: + get_rabbit_connection(True) + assert "Broker: No password provided for RabbitMQ" in str(excinfo.value) + + def test_get_rabbit_connection_invalid_pass_filepath(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_rabbit_connection` function with an invalid password filepath. + This should raise a ValueError. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.password = "invalid_filepath" + expanded_filepath = os.path.abspath(os.path.expanduser(CONFIG.broker.password)) + with pytest.raises(ValueError) as excinfo: + get_rabbit_connection(True) + assert f"Broker: RabbitMQ password file {expanded_filepath} does not exist" in str(excinfo.value) + + def run_get_connection_string(self, expected_vals: Dict[str, Any]): + """ + Helper method to run the tests for the `get_connection_string`. + + :param expected_vals: A dict of expected values for this test. Format: + {"conn": "", + "vhost": "host4testing", + "username": "default", + "password": "", + "server": "127.0.0.1", + "port": } + """ + expected = RABBITMQ_CONNECTION.format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_connection_string_rabbitmq(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rabbitmq as the broker. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "conn": "amqps", + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5671, + } + self.run_get_connection_string(expected_vals) + + def test_get_connection_string_amqp(self, rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with amqp as the broker. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + CONFIG.broker.name = "amqp" + expected_vals = { + "conn": "amqp", + "vhost": "host4testing", + "username": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + "port": 5672, + } + self.run_get_connection_string(expected_vals) + + +class TestRedisBroker: + """ + This class will house all tests necessary for our broker module when using a + redis broker. + """ + + def run_get_redissock_connection(self, expected_vals: Dict[str, str]): + """ + Helper method to run the tests for the `get_redissock_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"db_num": "", "path": ""} + """ + expected = REDISSOCK_CONNECTION.format(**expected_vals) + actual = get_redissock_connection() + assert actual == expected + + def test_get_redissock_connection(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with both a db_num and a broker path set. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Create and store a fake path and db_num for testing + test_path = "/fake/path/to/broker" + test_db_num = "45" + CONFIG.broker.path = test_path + CONFIG.broker.db_num = test_db_num + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": test_db_num, "path": test_path} + self.run_get_redissock_connection(expected_vals) + + def test_get_redissock_connection_no_db(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with a broker path set but no db num. + This should default the db_num to 0. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Create and store a fake path for testing + test_path = "/fake/path/to/broker" + CONFIG.broker.path = test_path + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": 0, "path": test_path} + self.run_get_redissock_connection(expected_vals) + + def test_get_redissock_connection_no_path(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with a db num set but no broker path. + This should raise an AttributeError since there will be no path value to read from + in `CONFIG.broker`. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.db_num = "45" + with pytest.raises(AttributeError): + get_redissock_connection() + + def test_get_redissock_connection_no_path_nor_db(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redissock_connection` function with neither a broker path nor a db num set. + This should raise an AttributeError since there will be no path value to read from + in `CONFIG.broker`. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + with pytest.raises(AttributeError): + get_redissock_connection() + + def run_get_redis_connection(self, expected_vals: Dict[str, Any], include_password: bool, use_ssl: bool): + """ + Helper method to run the tests for the `get_redis_connection`. + + :param expected_vals: A dict of expected values for this test. Format: + {"urlbase": "", "spass": "", "server": "127.0.0.1", "port": , "db_num": } + :param include_password: If True, include the password in the output. Otherwise don't. + :param use_ssl: If True, use ssl for the connection. Otherwise don't. + """ + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_redis_connection(include_password=include_password, use_ssl=use_ssl) + assert expected == actual + + def test_get_redis_connection(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_no_port(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the port setting from the CONFIG object. This should still run and give us + port = 6379. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.port + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_with_db(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after adding the db_num setting to the CONFIG object. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + test_db_num = "45" + CONFIG.broker.db_num = test_db_num + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": test_db_num, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_no_username(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the username setting from the CONFIG object. This should still run and give us + username = ''. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.username + expected_vals = {"urlbase": "redis", "spass": ":merlin-test-server@", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_redis_connection_invalid_pass_file(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after changing the permissions of the password file so it can't be opened. This should still + run and give us password = CONFIG.broker.password. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Capture the initial permissions of the password file so we can reset them + orig_file_permissions = os.stat(CONFIG.broker.password).st_mode + + # Change the permissions of the password file so it can't be read + os.chmod(CONFIG.broker.password, 0o222) + + try: + # Run the test + expected_vals = { + "urlbase": "redis", + "spass": f"default:{CONFIG.broker.password}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + except AssertionError as exc: + # If this test failed, make sure to reset the permissions in case other tests need to read this file + os.chmod(CONFIG.broker.password, orig_file_permissions) + raise AssertionError from exc + + os.chmod(CONFIG.broker.password, orig_file_permissions) + + def test_get_redis_connection_dont_include_password(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function without including the password. This should place 6 *s + where the password would normally be placed in spass. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = {"urlbase": "redis", "spass": "default:******@", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=False, use_ssl=False) + + def test_get_redis_connection_use_ssl(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with using ssl. This should change the urlbase to rediss (with two 's'). + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "rediss", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=True) + + def test_get_redis_connection_no_password(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_redis_connection` function with default functionality (including password and not using ssl). + We'll run this after deleting the password setting from the CONFIG object. This should still run and give us + spass = ''. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.broker.password + expected_vals = {"urlbase": "redis", "spass": "", "server": "127.0.0.1", "port": 6379, "db_num": 0} + self.run_get_redis_connection(expected_vals=expected_vals, include_password=True, use_ssl=False) + + def test_get_connection_string_redis(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis as the broker (this is what our CONFIG + is set to by default with the redis_broker_config fixture). + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert expected == actual + + def test_get_connection_string_rediss(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rediss (with two 's') as the broker. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "rediss" + expected_vals = { + "urlbase": "rediss", + "spass": "default:merlin-test-server@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert expected == actual + + def test_get_connection_string_redis_socket(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis+socket as the broker. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + # Change our broker + CONFIG.broker.name = "redis+socket" + + # Create and store a fake path and db_num for testing + test_path = "/fake/path/to/broker" + test_db_num = "45" + CONFIG.broker.path = test_path + CONFIG.broker.db_num = test_db_num + + # Set up our expected vals and compare against the actual result + expected_vals = {"db_num": test_db_num, "path": test_path} + expected = REDISSOCK_CONNECTION.format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_ssl_config_redis(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with redis as the broker (this is the default in our tests). + This should return False. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert not get_ssl_config() + + def test_get_ssl_config_rediss(self, redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss (with two 's') as the broker. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "rediss" + expected = {"ssl_cert_reqs": CERT_NONE} + actual = get_ssl_config() + assert actual == expected diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py new file mode 100644 index 000000000..64e56b7d9 --- /dev/null +++ b/tests/unit/config/test_config_object.py @@ -0,0 +1,150 @@ +""" +Test the functionality of the Config object. +""" + +from copy import copy, deepcopy +from types import SimpleNamespace + +from merlin.config import Config + + +class TestConfig: + """ + Class for testing the Config object. We'll store a valid `app_dict` + as an attribute here so that each test doesn't have to redefine it + each time. + """ + + app_dict = { + "celery": {"override": {"visibility_timeout": 86400}}, + "broker": { + "cert_reqs": "none", + "name": "rabbitmq", + "password": "/path/to/pass_file", + "port": 5671, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + }, + "results_backend": { + "cert_reqs": "none", + "db_num": 0, + "name": "rediss", + "password": "/path/to/pass_file", + "port": 6379, + "server": "127.0.0.1", + "username": "default", + "vhost": "host4testing", + "encryption_key": "/path/to/encryption_key", + }, + } + + def test_config_creation(self): + """ + Test the creation of the Config object. This should create nested namespaces + for each key in the `app_dict` variable and save them to their respective + attributes in the object. + """ + config = Config(self.app_dict) + + # Create the nested namespace for celery and compare result + override_namespace = SimpleNamespace(**self.app_dict["celery"]["override"]) + updated_celery_dict = deepcopy(self.app_dict) + updated_celery_dict["celery"]["override"] = override_namespace + celery_namespace = SimpleNamespace(**updated_celery_dict["celery"]) + assert config.celery == celery_namespace + + # Broker and Results Backend are easier since there's no nested namespace here + assert config.broker == SimpleNamespace(**self.app_dict["broker"]) + assert config.results_backend == SimpleNamespace(**self.app_dict["results_backend"]) + + def test_config_creation_no_celery(self): + """ + Test the creation of the Config object without the celery key. This should still + work and just not set anything for the celery attribute. + """ + + # Copy the celery section so we can restore it later and then delete it + celery_section = copy(self.app_dict["celery"]) + del self.app_dict["celery"] + config = Config(self.app_dict) + + # Broker and Results Backend are the only things loaded here + assert config.broker == SimpleNamespace(**self.app_dict["broker"]) + assert config.results_backend == SimpleNamespace(**self.app_dict["results_backend"]) + + # Ensure the celery attribute is not loaded + assert "celery" not in dir(config) + + # Reset celery section in case other tests use it after this + self.app_dict["celery"] = celery_section + + def test_config_copy(self): + """ + Test the `__copy__` magic method of the Config object. Here we'll make sure + each attribute was copied properly but the ids should be different. + """ + orig_config = Config(self.app_dict) + copied_config = copy(orig_config) + + assert orig_config.celery == copied_config.celery + assert orig_config.broker == copied_config.broker + assert orig_config.results_backend == copied_config.results_backend + + assert id(orig_config) != id(copied_config) + + def test_config_str(self): + """ + Test the `__str__` magic method of the Config object. This should just give us + a formatted string of the attributes in the object. + """ + config = Config(self.app_dict) + + # Test normal printing + actual = config.__str__() + expected = ( + "config:\n" + " celery:\n" + " override: namespace(visibility_timeout=86400)\n" + " broker:\n" + " cert_reqs: 'none'\n" + " name: 'rabbitmq'\n" + " password: '/path/to/pass_file'\n" + " port: 5671\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " results_backend:\n" + " cert_reqs: 'none'\n" + " db_num: 0\n" + " name: 'rediss'\n" + " password: '/path/to/pass_file'\n" + " port: 6379\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " encryption_key: '/path/to/encryption_key'" + ) + + assert actual == expected + + # Test printing with one section set to None + config.results_backend = None + actual_with_none = config.__str__() + expected_with_none = ( + "config:\n" + " celery:\n" + " override: namespace(visibility_timeout=86400)\n" + " broker:\n" + " cert_reqs: 'none'\n" + " name: 'rabbitmq'\n" + " password: '/path/to/pass_file'\n" + " port: 5671\n" + " server: '127.0.0.1'\n" + " username: 'default'\n" + " vhost: 'host4testing'\n" + " results_backend:\n" + " None" + ) + + assert actual_with_none == expected_with_none diff --git a/tests/unit/config/test_configfile.py b/tests/unit/config/test_configfile.py new file mode 100644 index 000000000..975e19ee4 --- /dev/null +++ b/tests/unit/config/test_configfile.py @@ -0,0 +1,696 @@ +""" +Tests for the configfile.py module. +""" + +import getpass +import os +import shutil +import ssl +from copy import copy, deepcopy + +import pytest +import yaml + +from merlin.config.configfile import ( + CONFIG, + default_config_info, + find_config_file, + get_cert_file, + get_config, + get_ssl_entries, + is_debug, + load_config, + load_default_celery, + load_default_user_names, + load_defaults, + merge_sslmap, + process_ssl_map, +) +from tests.constants import CERT_FILES +from tests.utils import create_dir + + +CONFIGFILE_DIR = "{temp_output_dir}/test_configfile" +COPIED_APP_FILENAME = "app_copy.yaml" +DUMMY_APP_FILEPATH = f"{os.path.dirname(__file__)}/dummy_app.yaml" + + +def create_app_yaml(app_yaml_filepath: str): + """ + Create a dummy app.yaml file at `app_yaml_filepath`. + + :param app_yaml_filepath: The location to create an app.yaml file at + """ + full_app_yaml_filepath = f"{app_yaml_filepath}/app.yaml" + if not os.path.exists(full_app_yaml_filepath): + shutil.copy(DUMMY_APP_FILEPATH, full_app_yaml_filepath) + + +def test_load_config(temp_output_dir: str): + """ + Test the `load_config` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) + create_app_yaml(configfile_dir) + + with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: + expected = yaml.load(dummy_app_file, yaml.Loader) + + actual = load_config(f"{configfile_dir}/app.yaml") + assert actual == expected + + +def test_load_config_invalid_file(): + """ + Test the `load_config` function with an invalid filepath. + """ + assert load_config("invalid/filepath") is None + + +def test_find_config_file_valid_path(temp_output_dir: str): + """ + Test the `find_config_file` function with passing a valid path in. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) + create_app_yaml(configfile_dir) + + assert find_config_file(configfile_dir) == f"{configfile_dir}/app.yaml" + + +def test_find_config_file_invalid_path(): + """ + Test the `find_config_file` function with passing an invalid path in. + """ + assert find_config_file("invalid/path") is None + + +def test_find_config_file_local_path(temp_output_dir: str): + """ + Test the `find_config_file` function by having it find a local (in our cwd) app.yaml file. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) + create_app_yaml(configfile_dir) + + # Move into the configfile directory and run the test + os.chdir(configfile_dir) + try: + assert find_config_file() == f"{os.getcwd()}/app.yaml" + except AssertionError as exc: + # Move back to the temp output directory even if the test fails + os.chdir(temp_output_dir) + raise AssertionError from exc + + # Move back to the temp output directory + os.chdir(temp_output_dir) + + +def test_find_config_file_merlin_home_path(temp_output_dir: str): + """ + Test the `find_config_file` function by having it find an app.yaml file in our merlin directory. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + merlin_home = os.path.expanduser("~/.merlin") + if not os.path.exists(merlin_home): + os.mkdir(merlin_home) + create_app_yaml(merlin_home) + assert find_config_file() == f"{merlin_home}/app.yaml" + + +def check_for_and_move_app_yaml(dir_to_check: str) -> bool: + """ + Check for any app.yaml files in `dir_to_check`. If one is found, rename it. + Return True if an app.yaml was found, false otherwise. + + :param dir_to_check: The directory to search for an app.yaml in + :returns: True if an app.yaml was found. False otherwise. + """ + for filename in os.listdir(dir_to_check): + full_path = os.path.join(dir_to_check, filename) + if os.path.isfile(full_path) and filename == "app.yaml": + os.rename(full_path, f"{dir_to_check}/{COPIED_APP_FILENAME}") + return True + return False + + +def test_find_config_file_no_path(temp_output_dir: str): + """ + Test the `find_config_file` function by making it unable to find any app.yaml path. + We'll use the `temp_output_dir` fixture so that our current working directory is in a temp + location. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Rename any app.yaml in the cwd + cwd_path = os.getcwd() + cwd_had_app_yaml = check_for_and_move_app_yaml(cwd_path) + + # Rename any app.yaml in the merlin home directory + merlin_home_dir = os.path.expanduser("~/.merlin") + merlin_home_had_app_yaml = check_for_and_move_app_yaml(merlin_home_dir) + + try: + assert find_config_file() is None + except AssertionError as exc: + # Reset the cwd app.yaml even if the test fails + if cwd_had_app_yaml: + os.rename(f"{cwd_path}/{COPIED_APP_FILENAME}", f"{cwd_path}/app.yaml") + + # Reset the merlin home app.yaml even if the test fails + if merlin_home_had_app_yaml: + os.rename(f"{merlin_home_dir}/{COPIED_APP_FILENAME}", f"{merlin_home_dir}/app.yaml") + + raise AssertionError from exc + + # Reset the cwd app.yaml + if cwd_had_app_yaml: + os.rename(f"{cwd_path}/{COPIED_APP_FILENAME}", f"{cwd_path}/app.yaml") + + # Reset the merlin home app.yaml + if merlin_home_had_app_yaml: + os.rename(f"{merlin_home_dir}/{COPIED_APP_FILENAME}", f"{merlin_home_dir}/app.yaml") + + +def test_load_default_user_names_nothing_to_load(): + """ + Test the `load_default_user_names` function with nothing to load. In other words, in this + test the config dict will have a username and vhost already set for the broker. We'll + create the dict then make a copy of it to test against after calling the function. + """ + actual_config = {"broker": {"username": "default", "vhost": "host4testing"}} + expected_config = deepcopy(actual_config) + assert actual_config is not expected_config + + load_default_user_names(actual_config) + + # Ensure that nothing was modified after our call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_user_names_no_username(): + """ + Test the `load_default_user_names` function with no username. In other words, in this + test the config dict will have vhost already set for the broker but not a username. + """ + expected_config = {"broker": {"username": getpass.getuser(), "vhost": "host4testing"}} + actual_config = {"broker": {"vhost": "host4testing"}} + load_default_user_names(actual_config) + + # Ensure that the username was set in the call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_user_names_no_vhost(): + """ + Test the `load_default_user_names` function with no vhost. In other words, in this + test the config dict will have username already set for the broker but not a vhost. + """ + expected_config = {"broker": {"username": "default", "vhost": getpass.getuser()}} + actual_config = {"broker": {"username": "default"}} + load_default_user_names(actual_config) + + # Ensure that the vhost was set in the call to load_default_user_names + assert actual_config == expected_config + + +def test_load_default_celery_nothing_to_load(): + """ + Test the `load_default_celery` function with nothing to load. In other words, in this + test the config dict will have a celery entry containing omit_queue_tag, queue_tag, and + override. We'll create the dict then make a copy of it to test against after calling + the function. + """ + actual_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + expected_config = deepcopy(actual_config) + assert actual_config is not expected_config + + load_default_celery(actual_config) + + # Ensure that nothing was modified after our call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_omit_queue_tag(): + """ + Test the `load_default_celery` function with no omit_queue_tag. The function should + create a default entry of False for this. + """ + actual_config = {"celery": {"queue_tag": "[merlin]_", "override": None}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the omit_queue_tag was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_queue_tag(): + """ + Test the `load_default_celery` function with no queue_tag. The function should + create a default entry of '[merlin]_' for this. + """ + actual_config = {"celery": {"omit_queue_tag": False, "override": None}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the queue_tag was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_override(): + """ + Test the `load_default_celery` function with no override. The function should + create a default entry of None for this. + """ + actual_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_"}} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the override was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_default_celery_no_celery_block(): + """ + Test the `load_default_celery` function with no celery block. The function should + create a default entry of + {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} for this. + """ + actual_config = {} + expected_config = {"celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}} + load_default_celery(actual_config) + + # Ensure that the celery block was set in the call to load_default_celery + assert actual_config == expected_config + + +def test_load_defaults(): + """ + Test that the `load_defaults` function loads the user names and the celery block properly. + """ + actual_config = {"broker": {}} + expected_config = { + "broker": {"username": getpass.getuser(), "vhost": getpass.getuser()}, + "celery": {"omit_queue_tag": False, "queue_tag": "[merlin]_", "override": None}, + } + load_defaults(actual_config) + + assert actual_config == expected_config + + +def test_get_config(temp_output_dir: str): + """ + Test the `get_config` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) + create_app_yaml(configfile_dir) + + # Load up the contents of the dummy app.yaml file that we copied + with open(DUMMY_APP_FILEPATH, "r") as dummy_app_file: + expected = yaml.load(dummy_app_file, yaml.Loader) + + # Add in default settings that should be added + expected["celery"]["omit_queue_tag"] = False + expected["celery"]["queue_tag"] = "[merlin]_" + + actual = get_config(configfile_dir) + + assert actual == expected + + +def test_get_config_invalid_path(): + """ + Test the `get_config` function with an invalid path. This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + get_config("invalid/path") + + assert "Cannot find a merlin config file!" in str(excinfo.value) + + +def test_is_debug_no_merlin_debug(): + """ + Test the `is_debug` function without having MERLIN_DEBUG in the environment. + This should return False. + """ + + # Delete the current val of MERLIN_DEBUG and store it (if there is one) + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + del os.environ["MERLIN_DEBUG"] + reset_merlin_debug = True + + # Run the test + try: + assert is_debug() is False + except AssertionError as exc: + # Make sure to reset the value of MERLIN_DEBUG even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + raise AssertionError from exc + + # Reset the value of MERLIN_DEBUG + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + + +def test_is_debug_with_merlin_debug(): + """ + Test the `is_debug` function with having MERLIN_DEBUG in the environment. + This should return True. + """ + + # Grab the current value of MERLIN_DEBUG if there is one + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ and int(os.environ["MERLIN_DEBUG"]) != 1: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + reset_merlin_debug = True + + # Set the MERLIN_DEBUG value to be 1 + os.environ["MERLIN_DEBUG"] = "1" + + try: + assert is_debug() is True + except AssertionError as exc: + # Make sure to reset the value of MERLIN_DEBUG even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + raise AssertionError from exc + + # Reset the value of MERLIN_DEBUG + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + + +def test_default_config_info(temp_output_dir: str): + """ + Test the `default_config_info` function. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + + # Create the configfile directory and put an app.yaml file there + configfile_dir = CONFIGFILE_DIR.format(temp_output_dir=temp_output_dir) + create_dir(configfile_dir) + create_app_yaml(configfile_dir) + cwd = os.getcwd() + os.chdir(configfile_dir) + + # Delete the current val of MERLIN_DEBUG and store it (if there is one) + reset_merlin_debug = False + debug_val = None + if "MERLIN_DEBUG" in os.environ: + debug_val = copy(os.environ["MERLIN_DEBUG"]) + del os.environ["MERLIN_DEBUG"] + reset_merlin_debug = True + + # Create the merlin home directory if it doesn't already exist + merlin_home = f"{os.path.expanduser('~')}/.merlin" + remove_merlin_home = False + if not os.path.exists(merlin_home): + os.mkdir(merlin_home) + remove_merlin_home = True + + # Run the test + try: + expected = { + "config_file": f"{configfile_dir}/app.yaml", + "is_debug": False, + "merlin_home": merlin_home, + "merlin_home_exists": True, + } + actual = default_config_info() + assert actual == expected + except AssertionError as exc: + # Make sure to reset values even if the test fails + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + if remove_merlin_home: + os.rmdir(merlin_home) + raise AssertionError from exc + + # Reset values if necessary + if reset_merlin_debug: + os.environ["MERLIN_DEBUG"] = debug_val + if remove_merlin_home: + os.rmdir(merlin_home) + + os.chdir(cwd) + + +def test_get_cert_file_all_valid_args(mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_cert_file` function with all valid arguments. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + expected = f"{merlin_server_dir}/{CERT_FILES['ssl_key']}" + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="keyfile", cert_path=merlin_server_dir + ) + assert actual == expected + + +def test_get_cert_file_invalid_cert_name(mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_cert_file` function with an invalid cert_name argument. This should just return None. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="invalid", cert_path=merlin_server_dir + ) + assert actual is None + + +def test_get_cert_file_nonexistent_cert_path( + mysql_results_backend_config: "fixture", temp_output_dir: str, merlin_server_dir: str # noqa: F821 +): + """ + Test the `get_cert_file` function with cert_path argument that doesn't exist. + This should still return the nonexistent path at the root of our temporary directory for testing. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param merlin_server_dir: The path to the temporary merlin server directory that's housing our cert files + """ + CONFIG.results_backend.certfile = "new_certfile.pem" + expected = f"{temp_output_dir}/new_certfile.pem" + actual = get_cert_file( + server_type="Results Backend", config=CONFIG.results_backend, cert_name="certfile", cert_path=merlin_server_dir + ) + assert actual == expected + + +def test_get_ssl_entries_required_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll make + cert reqs be required. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "required" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_REQUIRED, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_optional_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll make + cert reqs be optional. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "optional" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_OPTIONAL, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_none_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we won't require + any cert reqs. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + CONFIG.results_backend.cert_reqs = "none" + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_NONE, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_omit_certs(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll completely + omit the cert_reqs option + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + del CONFIG.results_backend.cert_reqs + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_REQUIRED, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_get_ssl_entries_with_ssl_protocol(mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_entries` function with mysql as the results_backend. For this test we'll add in a + dummy ssl_protocol value that should get added to the dict that's output. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + protocol = "test_protocol" + CONFIG.results_backend.ssl_protocol = protocol + + expected = { + "ssl_key": f"{temp_output_dir}/{CERT_FILES['ssl_key']}", + "ssl_cert": f"{temp_output_dir}/{CERT_FILES['ssl_cert']}", + "ssl_ca": f"{temp_output_dir}/{CERT_FILES['ssl_ca']}", + "cert_reqs": ssl.CERT_NONE, + "ssl_protocol": protocol, + } + actual = get_ssl_entries( + server_type="Results Backend", server_name="mysql", server_config=CONFIG.results_backend, cert_path=temp_output_dir + ) + assert expected == actual + + +def test_process_ssl_map_mysql(): + """Test the `process_ssl_map` function with mysql as the server name.""" + expected = {"keyfile": "ssl_key", "certfile": "ssl_cert", "ca_certs": "ssl_ca"} + actual = process_ssl_map("mysql") + assert actual == expected + + +def test_process_ssl_map_rediss(): + """Test the `process_ssl_map` function with rediss as the server name.""" + expected = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = process_ssl_map("rediss") + assert actual == expected + + +def test_merge_sslmap_all_keys_present(): + """ + Test the `merge_sslmap` function with all keys from server_ssl in ssl_map. + We'll assume we're using a rediss server for this. + """ + expected = { + "ssl_keyfile": "/path/to/keyfile", + "ssl_certfile": "/path/to/certfile", + "ssl_ca_certs": "/path/to/ca_file", + "ssl_cert_reqs": ssl.CERT_NONE, + } + test_server_ssl = { + "keyfile": "/path/to/keyfile", + "certfile": "/path/to/certfile", + "ca_certs": "/path/to/ca_file", + "cert_reqs": ssl.CERT_NONE, + } + test_ssl_map = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = merge_sslmap(test_server_ssl, test_ssl_map) + assert actual == expected + + +def test_merge_sslmap_some_keys_present(): + """ + Test the `merge_sslmap` function with some keys from server_ssl in ssl_map and others not. + We'll assume we're using a rediss server for this. + """ + expected = { + "ssl_keyfile": "/path/to/keyfile", + "ssl_certfile": "/path/to/certfile", + "ssl_ca_certs": "/path/to/ca_file", + "ssl_cert_reqs": ssl.CERT_NONE, + "new_key": "new_val", + "second_new_key": "second_new_val", + } + test_server_ssl = { + "keyfile": "/path/to/keyfile", + "certfile": "/path/to/certfile", + "ca_certs": "/path/to/ca_file", + "cert_reqs": ssl.CERT_NONE, + "new_key": "new_val", + "second_new_key": "second_new_val", + } + test_ssl_map = { + "keyfile": "ssl_keyfile", + "certfile": "ssl_certfile", + "ca_certs": "ssl_ca_certs", + "cert_reqs": "ssl_cert_reqs", + } + actual = merge_sslmap(test_server_ssl, test_ssl_map) + assert actual == expected diff --git a/tests/unit/config/test_results_backend.py b/tests/unit/config/test_results_backend.py new file mode 100644 index 000000000..f49e3e897 --- /dev/null +++ b/tests/unit/config/test_results_backend.py @@ -0,0 +1,593 @@ +""" +Tests for the `results_backend.py` file. +""" + +import os +from ssl import CERT_NONE +from typing import Any, Dict + +import pytest + +from merlin.config.configfile import CONFIG +from merlin.config.results_backend import ( + MYSQL_CONFIG_FILENAMES, + MYSQL_CONNECTION_STRING, + SQLITE_CONNECTION_STRING, + get_backend_password, + get_connection_string, + get_mysql, + get_mysql_config, + get_redis, + get_ssl_config, +) +from tests.constants import CERT_FILES, SERVER_PASS +from tests.utils import create_cert_files, create_pass_file + + +RESULTS_BACKEND_DIR = "{temp_output_dir}/test_results_backend" + + +def test_get_backend_password_pass_file_in_merlin(): + """ + Test the `get_backend_password` function with the password file in the ~/.merlin/ + directory. We'll create a dummy file in this directory and delete it once the test + is done. + """ + + # Check if the .merlin directory exists and create it if it doesn't + remove_merlin_dir_after_test = False + path_to_merlin_dir = os.path.expanduser("~/.merlin") + if not os.path.exists(path_to_merlin_dir): + remove_merlin_dir_after_test = True + os.mkdir(path_to_merlin_dir) + + # Create the test password file + pass_filename = "test.pass" + full_pass_filepath = f"{path_to_merlin_dir}/{pass_filename}" + create_pass_file(full_pass_filepath) + + try: + # Run the test + assert get_backend_password(pass_filename) == SERVER_PASS + # Cleanup + os.remove(full_pass_filepath) + if remove_merlin_dir_after_test: + os.rmdir(path_to_merlin_dir) + except AssertionError as exc: + # If the test fails, make sure we clean up the files/dirs created + os.remove(full_pass_filepath) + if remove_merlin_dir_after_test: + os.rmdir(path_to_merlin_dir) + raise AssertionError from exc + + +def test_get_backend_password_pass_file_not_in_merlin(temp_output_dir: str): + """ + Test the `get_backend_password` function with the password file not in the ~/.merlin/ + directory. By using the `temp_output_dir` fixture, our cwd will be the temporary directory. + We'll create a password file in the this directory for this test and have `get_backend_password` + read from that. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + pass_file = "test.pass" + create_pass_file(pass_file) + + assert get_backend_password(pass_file) == SERVER_PASS + + +def test_get_backend_password_directly_pass_password(): + """ + Test the `get_backend_password` function by passing the password directly to this + function instead of a password file. + """ + assert get_backend_password(SERVER_PASS) == SERVER_PASS + + +def test_get_backend_password_using_certs_path(temp_output_dir: str): + """ + Test the `get_backend_password` function with certs_path set to our temporary testing path. + We'll create a password file in the temporary directory for this test and have `get_backend_password` + read from that. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + pass_filename = "test_certs.pass" + test_dir = RESULTS_BACKEND_DIR.format(temp_output_dir=temp_output_dir) + if not os.path.exists(test_dir): + os.mkdir(test_dir) + full_pass_filepath = f"{test_dir}/{pass_filename}" + create_pass_file(full_pass_filepath) + + assert get_backend_password(pass_filename, certs_path=test_dir) == SERVER_PASS + + +def test_get_ssl_config_no_results_backend(config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with no results_backend set. This should return False. + NOTE: we're using the config fixture here to make sure values are reset after this test finishes. + We won't actually use anything from the config fixture. + + :param config: A fixture to set up the CONFIG object for us + """ + del CONFIG.results_backend.name + assert get_ssl_config() is False + + +def test_get_connection_string_no_results_backend(config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with no results_backend set. + This should raise a ValueError. + NOTE: we're using the config fixture here to make sure values are reset after this test finishes. + We won't actually use anything from the config fixture. + + :param config: A fixture to set up the CONFIG object for us + """ + del CONFIG.results_backend.name + with pytest.raises(ValueError) as excinfo: + get_connection_string() + + assert "'' is not a supported results backend" in str(excinfo.value) + + +class TestRedisResultsBackend: + """ + This class will house all tests necessary for our results_backend module when using a + redis results_backend. + """ + + def run_get_redis( + self, + expected_vals: Dict[str, Any], + certs_path: str = None, + include_password: bool = True, + ssl: bool = False, + ): + """ + Helper method for running tests for the `get_redis` function. + + :param expected_vals: A dict of expected values for this test. Format: + {"urlbase": "redis", + "spass": "", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0} + :param certs_path: A string denoting the path to the certification files + :param include_password: If True, include the password in the output. Otherwise don't. + :param ssl: If True, use ssl. Otherwise, don't. + """ + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_redis(certs_path=certs_path, include_password=include_password, ssl=ssl) + assert actual == expected + + def test_get_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with default functionality. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_dont_include_password(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with the password hidden. This should * out the password. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": "default:******@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=False, ssl=False) + + def test_get_redis_using_ssl(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with ssl enabled. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "rediss", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=True) + + def test_get_redis_no_port(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no port in our CONFIG object. This should default to port=6379. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.port + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_db_num(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no db_num in our CONFIG object. This should default to db_num=0. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.db_num + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_username(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no username in our CONFIG object. This should default to username=''. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.username + expected_vals = { + "urlbase": "redis", + "spass": f":{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_no_password_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function with no password filepath in our CONFIG object. This should default to spass=''. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.password + expected_vals = { + "urlbase": "redis", + "spass": "", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + + def test_get_redis_invalid_pass_file(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_redis` function. We'll run this after changing the permissions of the password file so it + can't be opened. This should still run and give us password=CONFIG.results_backend.password. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + + # Capture the initial permissions of the password file so we can reset them + orig_file_permissions = os.stat(CONFIG.results_backend.password).st_mode + + # Change the permissions of the password file so it can't be read + os.chmod(CONFIG.results_backend.password, 0o222) + + try: + # Run the test + expected_vals = { + "urlbase": "redis", + "spass": f"default:{CONFIG.results_backend.password}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + self.run_get_redis(expected_vals=expected_vals, certs_path=None, include_password=True, ssl=False) + os.chmod(CONFIG.results_backend.password, orig_file_permissions) + except AssertionError as exc: + # If this test failed, make sure to reset the permissions in case other tests need to read this file + os.chmod(CONFIG.results_backend.password, orig_file_permissions) + raise AssertionError from exc + + def test_get_ssl_config_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with redis as the results_backend. This should return False since + ssl requires using rediss (with two 's'). + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config() is False + + def test_get_ssl_config_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss as the results_backend. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "rediss" + assert get_ssl_config() == {"ssl_cert_reqs": CERT_NONE} + + def test_get_ssl_config_rediss_no_cert_reqs(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with rediss as the results_backend and no cert_reqs set. + This should return True. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + del CONFIG.results_backend.cert_reqs + CONFIG.results_backend.name = "rediss" + assert get_ssl_config() is True + + def test_get_connection_string_redis(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with redis as the results_backend. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + expected_vals = { + "urlbase": "redis", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + def test_get_connection_string_rediss(self, redis_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with rediss as the results_backend. + + :param redis_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "rediss" + expected_vals = { + "urlbase": "rediss", + "spass": f"default:{SERVER_PASS}@", + "server": "127.0.0.1", + "port": 6379, + "db_num": 0, + } + expected = "{urlbase}://{spass}{server}:{port}/{db_num}".format(**expected_vals) + actual = get_connection_string() + assert actual == expected + + +class TestMySQLResultsBackend: + """ + This class will house all tests necessary for our results_backend module when using a + MySQL results_backend. + NOTE: You'll notice a lot of these tests are setting CONFIG.results_backend.name to be + "invalid". This is so that we can get by the first if statement in the `get_mysql_config` + function. + """ + + def test_get_mysql_config_certs_set(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql_config` function with the certs dict getting set and returned. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected = {} + for key, cert_file in CERT_FILES.items(): + expected[key] = f"{merlin_server_dir}/{cert_file}" + actual = get_mysql_config(merlin_server_dir, CERT_FILES) + assert actual == expected + + def test_get_mysql_config_ssl_exists(self, mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_mysql_config` function with mysql_ssl being found. This should just return the ssl value that's found. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + expected = {key: f"{temp_output_dir}/{cert_file}" for key, cert_file in CERT_FILES.items()} + expected["cert_reqs"] = CERT_NONE + assert get_mysql_config(None, None) == expected + + def test_get_mysql_config_no_mysql_certs( + self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 + ): + """ + Test the `get_mysql_config` function with no mysql certs dict. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + assert get_mysql_config(merlin_server_dir, {}) == {} + + def test_get_mysql_config_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql_config` function with an invalid certs path. This should return False. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "invalid" + assert get_mysql_config("invalid/path", CERT_FILES) is False + + def run_get_mysql( + self, expected_vals: Dict[str, Any], certs_path: str, mysql_certs: Dict[str, str], include_password: bool + ): + """ + Helper method for running tests for the `get_mysql` function. + + :param expected_vals: A dict of expected values for this test. Format: + {"cert_reqs": cert reqs dict, + "user": "default", + "password": "", + "server": "127.0.0.1", + "ssl_cert": "test-rabbit-client-cert.pem", + "ssl_ca": "test-mysql-ca-cert.pem", + "ssl_key": "test-rabbit-client-key.pem"} + :param certs_path: A string denoting the path to the certification files + :param mysql_certs: A dict of cert files + :param include_password: If True, include the password in the output. Otherwise don't. + """ + expected = MYSQL_CONNECTION_STRING.format(**expected_vals) + actual = get_mysql(certs_path=certs_path, mysql_certs=mysql_certs, include_password=include_password) + assert actual == expected + + def test_get_mysql(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function with default behavior. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + for key, cert_file in CERT_FILES.items(): + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + self.run_get_mysql( + expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=True + ) + + def test_get_mysql_dont_include_password( + self, mysql_results_backend_config: "fixture", merlin_server_dir: str # noqa: F821 + ): + """ + Test the `get_mysql` function but set include_password to False. This should * out the password. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": "******", + "server": "127.0.0.1", + } + for key, cert_file in CERT_FILES.items(): + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + self.run_get_mysql( + expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=CERT_FILES, include_password=False + ) + + def test_get_mysql_no_mysql_certs(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_mysql` function with no mysql_certs passed in. This should use default config filenames so we'll + have to create these default files. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.results_backend.name = "invalid" + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + + create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + + for key, cert_file in MYSQL_CONFIG_FILENAMES.items(): + # Password file is already is already set in expected_vals dict + if key == "password": + continue + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + + self.run_get_mysql(expected_vals=expected_vals, certs_path=merlin_server_dir, mysql_certs=None, include_password=True) + + def test_get_mysql_no_server(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql` function with no server set. This should raise a TypeError. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.server = False + with pytest.raises(TypeError) as excinfo: + get_mysql() + assert "Results backend: server False does not have a configuration" in str(excinfo.value) + + def test_get_mysql_invalid_certs_path(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_mysql` function with an invalid certs_path. This should raise a TypeError. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "invalid" + with pytest.raises(TypeError) as excinfo: + get_mysql(certs_path="invalid_path", mysql_certs=CERT_FILES) + err_msg = f"""The connection information for MySQL could not be set, cannot find:\n + {CERT_FILES}\ncheck the celery/certs path or set the ssl information in the app.yaml file.""" + assert err_msg in str(excinfo.value) + + def test_get_ssl_config_mysql(self, mysql_results_backend_config: "fixture", temp_output_dir: str): # noqa: F821 + """ + Test the `get_ssl_config` function with mysql as the results_backend. + This should return a dict of cert reqs with ssl.CERT_NONE as the value. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + """ + expected = {key: f"{temp_output_dir}/{cert_file}" for key, cert_file in CERT_FILES.items()} + expected["cert_reqs"] = CERT_NONE + assert get_ssl_config() == expected + + def test_get_ssl_config_mysql_celery_check(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_ssl_config` function with mysql as the results_backend and celery_check set. + This should return False. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_ssl_config(celery_check=True) is False + + def test_get_connection_string_mysql(self, mysql_results_backend_config: "fixture", merlin_server_dir: str): # noqa: F821 + """ + Test the `get_connection_string` function with MySQL as the results_backend. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + :param merlin_server_dir: The directory that has the test certification files + """ + CONFIG.celery.certs = merlin_server_dir + + create_cert_files(merlin_server_dir, MYSQL_CONFIG_FILENAMES) + CONFIG.results_backend.keyfile = MYSQL_CONFIG_FILENAMES["ssl_key"] + CONFIG.results_backend.certfile = MYSQL_CONFIG_FILENAMES["ssl_cert"] + CONFIG.results_backend.ca_certs = MYSQL_CONFIG_FILENAMES["ssl_ca"] + + expected_vals = { + "cert_reqs": CERT_NONE, + "user": "default", + "password": SERVER_PASS, + "server": "127.0.0.1", + } + for key, cert_file in MYSQL_CONFIG_FILENAMES.items(): + # Password file is already is already set in expected_vals dict + if key == "password": + continue + expected_vals[key] = f"{merlin_server_dir}/{cert_file}" + + assert MYSQL_CONNECTION_STRING.format(**expected_vals) == get_connection_string(include_password=True) + + def test_get_connection_string_sqlite(self, mysql_results_backend_config: "fixture"): # noqa: F821 + """ + Test the `get_connection_string` function with sqlite as the results_backend. + + :param mysql_results_backend_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.results_backend.name = "sqlite" + assert get_connection_string() == SQLITE_CONNECTION_STRING diff --git a/tests/unit/config/test_utils.py b/tests/unit/config/test_utils.py new file mode 100644 index 000000000..9d64c10c7 --- /dev/null +++ b/tests/unit/config/test_utils.py @@ -0,0 +1,117 @@ +""" +Tests for the merlin/config/utils.py module. +""" + +import pytest + +from merlin.config.configfile import CONFIG +from merlin.config.utils import Priority, determine_priority_map, get_priority, is_rabbit_broker, is_redis_broker + + +def test_is_rabbit_broker(): + """Test the `is_rabbit_broker` by passing in rabbit as the broker""" + assert is_rabbit_broker("rabbitmq") is True + assert is_rabbit_broker("amqp") is True + assert is_rabbit_broker("amqps") is True + + +def test_is_rabbit_broker_invalid(): + """Test the `is_rabbit_broker` by passing in an invalid broker""" + assert is_rabbit_broker("redis") is False + assert is_rabbit_broker("") is False + + +def test_is_redis_broker(): + """Test the `is_redis_broker` by passing in redis as the broker""" + assert is_redis_broker("redis") is True + assert is_redis_broker("rediss") is True + assert is_redis_broker("redis+socket") is True + + +def test_is_redis_broker_invalid(): + """Test the `is_redis_broker` by passing in an invalid broker""" + assert is_redis_broker("rabbitmq") is False + assert is_redis_broker("") is False + + +def test_get_priority_rabbit_broker(rabbit_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with rabbit as the broker. + Low priority for rabbit is 1 and high is 9. + + :param rabbit_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_priority(Priority.LOW) == 1 + assert get_priority(Priority.MID) == 5 + assert get_priority(Priority.HIGH) == 9 + assert get_priority(Priority.RETRY) == 10 + + +def test_get_priority_redis_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with redis as the broker. + Low priority for redis is 10 and high is 2. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + assert get_priority(Priority.LOW) == 10 + assert get_priority(Priority.MID) == 5 + assert get_priority(Priority.HIGH) == 2 + assert get_priority(Priority.RETRY) == 1 + + +def test_get_priority_invalid_broker(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with an invalid broker. + This should raise a ValueError. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + CONFIG.broker.name = "invalid" + with pytest.raises(ValueError) as excinfo: + get_priority(Priority.LOW) + assert "Unsupported broker name: invalid" in str(excinfo.value) + + +def test_get_priority_invalid_priority(redis_broker_config: "fixture"): # noqa: F821 + """ + Test the `get_priority` function with an invalid priority. + This should raise a TypeError. + + :param redis_broker_config: A fixture to set the CONFIG object to a test configuration that we'll use here + """ + with pytest.raises(ValueError) as excinfo: + get_priority("invalid_priority") + assert "Invalid priority: invalid_priority" in str(excinfo.value) + + +def test_determine_priority_map_rabbit(): + """ + Test the `determine_priority_map` function with rabbit as the broker. + This should return the following map: + {Priority.LOW: 1, Priority.MID: 5, Priority.HIGH: 9, Priority.RETRY: 10} + """ + expected = {Priority.LOW: 1, Priority.MID: 5, Priority.HIGH: 9, Priority.RETRY: 10} + actual = determine_priority_map("rabbitmq") + assert actual == expected + + +def test_determine_priority_map_redis(): + """ + Test the `determine_priority_map` function with redis as the broker. + This should return the following map: + {Priority.LOW: 10, Priority.MID: 5, Priority.HIGH: 2, Priority.RETRY: 1} + """ + expected = {Priority.LOW: 10, Priority.MID: 5, Priority.HIGH: 2, Priority.RETRY: 1} + actual = determine_priority_map("redis") + assert actual == expected + + +def test_determine_priority_map_invalid(): + """ + Test the `determine_priority_map` function with an invalid broker. + This should raise a ValueError. + """ + with pytest.raises(ValueError) as excinfo: + determine_priority_map("invalid_broker") + assert "Unsupported broker name: invalid_broker" in str(excinfo.value) diff --git a/tests/unit/config/utils.py b/tests/unit/config/utils.py deleted file mode 100644 index 1765e8478..000000000 --- a/tests/unit/config/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Utils module for common test functionality. -""" - -import os - - -def mkfile(tmpdir, filename, content=""): - """ - A simple function for creating a file and returning the path. This is to - abstract out file creation logic in the tests. - - :param tmpdir: (str) The path to the temp directory. - :param filename: (str) The name of the file. - :param contents: (str) Optional contents to write to the file. Defaults to - an empty string. - :returns: (str) The appended path of the given tempdir and filename. - """ - filepath = os.path.join(tmpdir, filename) - - with open(filepath, "w") as f: - f.write(content) - - return filepath diff --git a/tests/unit/server/__init__.py b/tests/unit/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/server/test_RedisConfig.py b/tests/unit/server/test_RedisConfig.py new file mode 100644 index 000000000..321d2f38a --- /dev/null +++ b/tests/unit/server/test_RedisConfig.py @@ -0,0 +1,556 @@ +""" +Tests for the RedisConfig class of the `server_util.py` module. + +This class is especially large so that's why these tests have been +moved to their own file. +""" + +import filecmp +import logging +from typing import Any + +import pytest + +from merlin.server.server_util import RedisConfig + + +class TestRedisConfig: + """Tests for the RedisConfig class.""" + + def test_initialization(self, server_redis_conf_file: str): + """ + Using a dummy redis configuration file, test that the initialization + of the RedisConfig class behaves as expected. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + expected_entries = { + "bind": "127.0.0.1", + "port": "6379", + "requirepass": "merlin_password", + "dir": "./", + "save": "300 100", + "dbfilename": "dump.rdb", + "appendfsync": "everysec", + "appendfilename": "appendonly.aof", + } + expected_comments = { + "bind": "# ip address\n", + "port": "\n# port\n", + "requirepass": "\n# password\n", + "dir": "\n# directory\n", + "save": "\n# snapshot\n", + "dbfilename": "\n# db file\n", + "appendfsync": "\n# append mode\n", + "appendfilename": "\n# append file\n", + } + expected_trailing_comment = "\n# dummy trailing comment" + expected_entry_order = list(expected_entries.keys()) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.filename == server_redis_conf_file + assert not redis_config.changed + assert redis_config.entries == expected_entries + assert redis_config.entry_order == expected_entry_order + assert redis_config.comments == expected_comments + assert redis_config.trailing_comments == expected_trailing_comment + + def test_write(self, server_redis_conf_file: str, server_testing_dir: str): + """ + Test that the write functionality works by writing the contents of a dummy + configuration file to a blank configuration file. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_redis_conf_file = f"{server_testing_dir}/redis_copy.conf" + + # Create a RedisConf object with the basic redis conf file + redis_config = RedisConfig(server_redis_conf_file) + + # Change the filepath of the redis config file to be the copy that we'll write to + redis_config.set_filename(copy_redis_conf_file) + + # Run the test + redis_config.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_conf_file, copy_redis_conf_file) + + @pytest.mark.parametrize("key, val, expected_return", [("port", 1234, True), ("invalid_key", "dummy_val", False)]) + def test_set_config_value(self, server_redis_conf_file: str, key: str, val: Any, expected_return: bool): + """ + Test the `set_config_value` method with valid and invalid keys. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param key: The key value to modify with `set_config_value` + :param val: The value to set `key` to + :param expected_return: The expected return from `set_config_value` + """ + redis_config = RedisConfig(server_redis_conf_file) + actual_return = redis_config.set_config_value(key, val) + assert actual_return == expected_return + if expected_return: + assert redis_config.entries[key] == val + assert redis_config.changes_made() + else: + assert not redis_config.changes_made() + + @pytest.mark.parametrize( + "key, expected_val", + [ + ("bind", "127.0.0.1"), + ("port", "6379"), + ("requirepass", "merlin_password"), + ("dir", "./"), + ("save", "300 100"), + ("dbfilename", "dump.rdb"), + ("appendfsync", "everysec"), + ("appendfilename", "appendonly.aof"), + ("invalid_key", None), + ], + ) + def test_get_config_value(self, server_redis_conf_file: str, key: str, expected_val: str): + """ + Test the `get_config_value` method with valid and invalid keys. + + :param server_redis_conf_file: The path to a dummy redis configuration file + :param key: The key value to modify with `set_config_value` + :param expected_val: The value we're expecting to get by querying `key` + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.get_config_value(key) == expected_val + + @pytest.mark.parametrize( + "ip_to_set", + [ + "127.0.0.1", # Most common IP + "0.0.0.0", # Edge case (low) + "255.255.255.255", # Edge case (high) + "123.222.199.20", # Random valid IP + ], + ) + def test_set_ip_address_valid(self, caplog: "Fixture", server_redis_conf_file: str, ip_to_set: str): # noqa: F821 + """ + Test the `set_ip_address` method with valid ips. These should all return True + and set the 'bind' value to whatever `ip_to_set` is. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param ip_to_set: The ip address to set + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.set_ip_address(ip_to_set) + assert f"Ipaddress is set to {ip_to_set}" in caplog.text, "Missing expected log message" + assert redis_config.get_ip_address() == ip_to_set + + @pytest.mark.parametrize( + "ip_to_set, expected_log", + [ + (None, None), # No IP + ("0.0.0", "Invalid IPv4 address given."), # Invalid IPv4 + ("bind-unset", "Unable to set ip address for redis config"), # Special invalid case where bind doesn't exist + ], + ) + def test_set_ip_address_invalid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + ip_to_set: str, + expected_log: str, + ): + """ + Test the `set_ip_address` method with invalid ips. These should all return False. + and not modify the 'bind' setting. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param ip_to_set: The ip address to set + :param expected_log: The string we're expecting the logger to log + """ + redis_config = RedisConfig(server_redis_conf_file) + # For the test where bind is unset, delete bind from dict and set new ip val to a valid value + if ip_to_set == "bind-unset": + del redis_config.entries["bind"] + ip_to_set = "127.0.0.1" + assert not redis_config.set_ip_address(ip_to_set) + assert redis_config.get_ip_address() != ip_to_set + if expected_log is not None: + assert expected_log in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize( + "port_to_set", + [ + 6379, # Most common port + 1, # Edge case (low) + 65535, # Edge case (high) + 12345, # Random valid port + ], + ) + def test_set_port_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + port_to_set: str, + ): + """ + Test the `set_port` method with valid ports. These should all return True + and set the 'port' value to whatever `port_to_set` is. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param port_to_set: The port to set + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + assert redis_config.set_port(port_to_set) + assert redis_config.get_port() == port_to_set + assert f"Port is set to {port_to_set}" in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize( + "port_to_set, expected_log", + [ + (None, None), # No port + (0, "Invalid port given."), # Edge case (low) + (65536, "Invalid port given."), # Edge case (high) + ("port-unset", "Unable to set port for redis config"), # Special invalid case where port doesn't exist + ], + ) + def test_set_port_invalid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + port_to_set: str, + expected_log: str, + ): + """ + Test the `set_port` method with invalid inputs. These should all return False + and not modify the 'port' setting. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param port_to_set: The port to set + :param expected_log: The string we're expecting the logger to log + """ + redis_config = RedisConfig(server_redis_conf_file) + # For the test where port is unset, delete port from dict and set port val to a valid value + if port_to_set == "port-unset": + del redis_config.entries["port"] + port_to_set = 5 + assert not redis_config.set_port(port_to_set) + assert redis_config.get_port() != port_to_set + if expected_log is not None: + assert expected_log in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize( + "pass_to_set, expected_return", + [ + ("valid_password", True), # Valid password + (None, False), # Invalid password + ], + ) + def test_set_password( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + pass_to_set: str, + expected_return: bool, + ): + """ + Test the `set_password` method with both valid and invalid input. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param pass_to_set: The password to set + :param expected_return: The expected return value + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.set_password(pass_to_set) == expected_return + if expected_return: + assert redis_conf.get_password() == pass_to_set + assert "New password set" in caplog.text, "Missing expected log message" + + def test_set_directory_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + server_testing_dir: str, + ): + """ + Test the `set_directory` method with valid input. This should return True, modify the + 'dir' value, and log some messages about creating/setting the directory. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + caplog.set_level(logging.INFO) + redis_config = RedisConfig(server_redis_conf_file) + dir_to_set = f"{server_testing_dir}/dummy_dir" + assert redis_config.set_directory(dir_to_set) + assert redis_config.get_config_value("dir") == dir_to_set + assert f"Created directory {dir_to_set}" in caplog.text, "Missing created log message" + assert f"Directory is set to {dir_to_set}" in caplog.text, "Missing set log message" + + def test_set_directory_none(self, server_redis_conf_file: str): + """ + Test the `set_directory` method with None as the input. This should return False + and not modify the 'dir' setting. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_config = RedisConfig(server_redis_conf_file) + assert not redis_config.set_directory(None) + assert redis_config.get_config_value("dir") is not None + + def test_set_directory_dir_unset( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + server_testing_dir: str, + ): + """ + Test the `set_directory` method with the 'dir' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + redis_config = RedisConfig(server_redis_conf_file) + del redis_config.entries["dir"] + dir_to_set = f"{server_testing_dir}/dummy_dir" + assert not redis_config.set_directory(dir_to_set) + assert "Unable to set directory for redis config" in caplog.text, "Missing expected log message" + + def test_set_snapshot_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'seconds' and 'changes'. + This should return True and modify both values of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + snap_sec_to_set = 20 + snap_changes_to_set = 30 + assert redis_conf.set_snapshot(seconds=snap_sec_to_set, changes=snap_changes_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == str(snap_sec_to_set) + assert save_val[1] == str(snap_changes_to_set) + expected_log = ( + f"Snapshot wait time is set to {snap_sec_to_set} seconds. " + f"Snapshot threshold is set to {snap_changes_to_set} changes" + ) + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_just_seconds(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'seconds'. This should + return True and modify the first value of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + orig_save = redis_conf.get_config_value("save").split() + snap_sec_to_set = 20 + assert redis_conf.set_snapshot(seconds=snap_sec_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == str(snap_sec_to_set) + assert save_val[1] == orig_save[1] + expected_log = f"Snapshot wait time is set to {snap_sec_to_set} seconds. " + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_just_changes(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with a valid input for 'changes'. This should + return True and modify the second value of 'save'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + orig_save = redis_conf.get_config_value("save").split() + snap_changes_to_set = 30 + assert redis_conf.set_snapshot(changes=snap_changes_to_set) + save_val = redis_conf.get_config_value("save").split() + assert save_val[0] == orig_save[0] + assert save_val[1] == str(snap_changes_to_set) + expected_log = f"Snapshot threshold is set to {snap_changes_to_set} changes" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_snapshot_none(self, server_redis_conf_file: str): + """ + Test the `set_snapshot` method with None as the input for both seconds + and changes. This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_snapshot(seconds=None, changes=None) + + def test_set_snapshot_save_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with the 'save' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["save"] + assert not redis_conf.set_snapshot(seconds=20) + assert "Unable to get exisiting parameter values for snapshot" in caplog.text, "Missing expected log message" + + def test_set_snapshot_file_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot_file` method with a valid input. This should + return True and modify the value of 'dbfilename'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + filename = "dummy_file.rdb" + assert redis_conf.set_snapshot_file(filename) + assert redis_conf.get_config_value("dbfilename") == filename + assert f"Snapshot file is set to {filename}" in caplog.text, "Missing expected log message" + + def test_set_snapshot_file_none(self, server_redis_conf_file: str): + """ + Test the `set_snapshot_file` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_snapshot_file(None) + + def test_set_snapshot_file_dbfilename_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_snapshot` method with the 'dbfilename' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["dbfilename"] + filename = "dummy_file.rdb" + assert not redis_conf.set_snapshot_file(filename) + assert redis_conf.get_config_value("dbfilename") != filename + assert "Unable to set snapshot_file name" in caplog.text, "Missing expected log message" + + @pytest.mark.parametrize( + "mode_to_set", + [ + "always", + "everysec", + "no", + ], + ) + def test_set_append_mode_valid( + self, + caplog: "Fixture", # noqa: F821 + server_redis_conf_file: str, + mode_to_set: str, + ): + """ + Test the `set_append_mode` method with valid modes. These should all return True + and modify the value of 'appendfsync'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + :param mode_to_set: The mode to set + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + assert redis_conf.set_append_mode(mode_to_set) + assert redis_conf.get_config_value("appendfsync") == mode_to_set + assert f"Append mode is set to {mode_to_set}" in caplog.text, "Missing expected log message" + + def test_set_append_mode_invalid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_mode` method with an invalid mode. This should return False + and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + invalid_mode = "invalid" + assert not redis_conf.set_append_mode(invalid_mode) + assert redis_conf.get_config_value("appendfsync") != invalid_mode + expected_log = "Not a valid append_mode (Only valid modes are always, everysec, no)" + assert expected_log in caplog.text, "Missing expected log message" + + def test_set_append_mode_none(self, server_redis_conf_file: str): + """ + Test the `set_append_mode` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_append_mode(None) + + def test_set_append_mode_appendfsync_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_mode` method with the 'appendfsync' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["appendfsync"] + mode = "no" + assert not redis_conf.set_append_mode(mode) + assert redis_conf.get_config_value("appendfsync") != mode + assert "Unable to set append_mode in redis config" in caplog.text, "Missing expected log message" + + def test_set_append_file_valid(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_file` method with a valid file. This should return True + and modify the value of 'appendfilename'. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + caplog.set_level(logging.INFO) + redis_conf = RedisConfig(server_redis_conf_file) + valid_file = "valid" + assert redis_conf.set_append_file(valid_file) + assert redis_conf.get_config_value("appendfilename") == f'"{valid_file}"' + assert f"Append file is set to {valid_file}" in caplog.text, "Missing expected log message" + + def test_set_append_file_none(self, server_redis_conf_file: str): + """ + Test the `set_append_file` method with None as the input. + This should return False. + + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + assert not redis_conf.set_append_file(None) + + def test_set_append_file_appendfilename_unset(self, caplog: "Fixture", server_redis_conf_file: str): # noqa: F821 + """ + Test the `set_append_file` method with the 'appendfilename' setting not existing. This should + return False and log an error message. + + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + redis_conf = RedisConfig(server_redis_conf_file) + del redis_conf.entries["appendfilename"] + filename = "valid_filename" + assert not redis_conf.set_append_file(filename) + assert redis_conf.get_config_value("appendfilename") != filename + assert "Unable to set append filename." in caplog.text, "Missing expected log message" diff --git a/tests/unit/server/test_server_commands.py b/tests/unit/server/test_server_commands.py new file mode 100644 index 000000000..ec52df2a0 --- /dev/null +++ b/tests/unit/server/test_server_commands.py @@ -0,0 +1,647 @@ +""" +Tests for the `server_commands.py` module. +""" + +import logging +import os +import subprocess +from argparse import Namespace +from typing import Dict, List + +import pytest + +from merlin.server.server_commands import ( + check_for_not_running_server, + config_server, + init_server, + restart_server, + server_started, + start_container, + start_server, + status_server, + stop_server, +) +from merlin.server.server_config import ServerStatus +from merlin.server.server_util import ServerConfig + + +def test_init_server_create_server_fail(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `init_server` function with `create_server_config` returning False. + This should log a failure message. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + caplog.set_level(logging.INFO) + create_server_mock = mocker.patch("merlin.server.server_commands.create_server_config", return_value=False) + init_server() + create_server_mock.assert_called_once() + assert "Merlin server initialization failed." in caplog.text + + +def test_init_server_create_server_success(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `init_server` function with `create_server_config` returning True. + This should log a sucess message. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + caplog.set_level(logging.INFO) + create_server_mock = mocker.patch("merlin.server.server_commands.create_server_config", return_value=True) + pull_server_mock = mocker.patch("merlin.server.server_commands.pull_server_image", return_value=True) + config_merlin_mock = mocker.patch("merlin.server.server_commands.config_merlin_server", return_value=True) + init_server() + create_server_mock.assert_called_once() + pull_server_mock.assert_called_once() + config_merlin_mock.assert_called_once() + assert "Merlin server initialization successful." in caplog.text + + +def test_config_server_no_server_config( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_config_server_args: Namespace, +): + """ + Test the `config_server` function with no server config. This should log an error + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_config_server_args: An argparse Namespace with args needed by `config_server` + """ + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=None) + assert not config_server(server_config_server_args) + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +@pytest.mark.parametrize( + "server_status, status_name", + [ + (ServerStatus.RUNNING, "running"), + (ServerStatus.NOT_RUNNING, "not_running"), + ], +) +def test_config_server_add_user_remove_user_success( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_config_server_args: Namespace, + server_server_config: Dict[str, Dict[str, str]], + server_status: ServerStatus, + status_name: str, +): + """ + Test the `config_server` function by adding and removing a user. This will be ran with and without + the server status being set to RUNNING. For each scenario we should expect: + - RUNNING -> RedisUsers.write and RedisUsers.apply_to_redis are both called twice + - NOT_RUNNING -> RedisUsers.write is called twice and RedisUsers.apply_to_redis is not called at all + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_config_server_args: An argparse Namespace with args needed by `config_server` + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_status: The server status for this test (either RUNNING or NOT_RUNNING) + :param status_name: The name of the status in string form so we can have unique users for each test + """ + caplog.set_level(logging.INFO) + + # Set up the add_user and remove_user calls to test + user_to_add_and_remove = f"test_config_server_modification_user_{status_name}" + server_config_server_args.add_user = [user_to_add_and_remove, "test_config_server_modification_password"] + server_config_server_args.remove_user = user_to_add_and_remove + + # Create mocks of the necessary calls for this function + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.apply_config_changes") + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + write_mock = mocker.patch("merlin.server.server_util.RedisUsers.write") + apply_to_redis_mock = mocker.patch("merlin.server.server_util.RedisUsers.apply_to_redis") + + # Run the test + expected_apply_calls = 2 if server_status == ServerStatus.RUNNING else 0 + assert config_server(server_config_server_args) is None + assert write_mock.call_count == 2 + assert apply_to_redis_mock.call_count == expected_apply_calls + assert f"Added user {user_to_add_and_remove} to merlin server" in caplog.text + assert f"Removed user {user_to_add_and_remove} to merlin server" in caplog.text + + +def test_config_server_add_user_remove_user_failure( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_config_server_args: Namespace, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `config_server` function by attempting to add a user that already exists (we do this through mock) + and removing a user that doesn't exist. This should run to completion but never call RedisUsers.write. It + should also log two error messages. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_config_server_args: An argparse Namespace with args needed by `config_server` + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + # Set up the add_user and remove_user calls to test (these users should never actually be added/removed) + user_to_add_and_remove = "test_config_user_not_ever_added" + server_config_server_args.add_user = [user_to_add_and_remove, "test_config_server_modification_password"] + server_config_server_args.remove_user = user_to_add_and_remove + + # Create mocks of the necessary calls for this function + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.apply_config_changes") + mocker.patch("merlin.server.server_util.RedisUsers.add_user", return_value=False) + write_mock = mocker.patch("merlin.server.server_util.RedisUsers.write") + + # Run the test + assert config_server(server_config_server_args) is None + assert write_mock.call_count == 0 + assert f"User '{user_to_add_and_remove}' already exisits within current users" in caplog.text + assert f"User '{user_to_add_and_remove}' doesn't exist within current users." in caplog.text + + +@pytest.mark.parametrize( + "server_status, expected_log_msgs", + [ + ( + ServerStatus.NOT_INITIALIZED, + ["Merlin server has not been initialized.", "Please initalize server by running 'merlin server init'"], + ), + ( + ServerStatus.MISSING_CONTAINER, + ["Unable to find server image.", "Ensure there is a .sif file in merlin server directory."], + ), + (ServerStatus.NOT_RUNNING, ["Merlin server is not running."]), + (ServerStatus.RUNNING, ["Merlin server is running."]), + ], +) +def test_status_server( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_status: ServerStatus, + expected_log_msgs: List[str], +): + """ + Test the `status_server` function to make sure it produces the correct logs for each status. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + :param expected_log_msgs: The logs we're expecting from this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + status_server() + for expected_log_msg in expected_log_msgs: + assert expected_log_msg in caplog.text + + +@pytest.mark.parametrize( + "server_status, expected_result, expected_log_msg", + [ + ( + ServerStatus.NOT_INITIALIZED, + False, + "Merlin server has not been intitialized. Please run 'merlin server init' first.", + ), + ( + ServerStatus.MISSING_CONTAINER, + False, + "Merlin server has not been intitialized. Please run 'merlin server init' first.", + ), + (ServerStatus.NOT_RUNNING, True, None), + ( + ServerStatus.RUNNING, + False, + """Merlin server already running. + Stop current server with 'merlin server stop' before attempting to start a new server.""", + ), + ], +) +def test_check_for_not_running_server( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_status: ServerStatus, + expected_result: bool, + expected_log_msg: str, +): + """ + Test the `check_for_not_running_server` function with different server statuses. + There should be a logged message for each status and the results we should expect are as + follows: + - NOT_RUNNING status should return True + - any other status should return False + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + :param expected_result: The expected result (T/F) for this test + :param expected_log_msg: The log we're expecting from this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + assert check_for_not_running_server() == expected_result + if expected_log_msg is not None: + assert expected_log_msg in caplog.text + + +def test_start_container_invalid_image_path( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `start_container` function with a nonexistent image path. + This should log an error and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + image_file = "nonexistent.image" + server_server_config["container"]["image"] = image_file + server_server_config["container"]["config"] = "start_container.config" + + # Create the config path so we ensure it exists + config_path = f"{server_testing_dir}/{server_server_config['container']['config']}" + with open(config_path, "w"): + pass + + assert start_container(ServerConfig(server_server_config)) is None + assert f"Unable to find image at {os.path.join(server_testing_dir, image_file)}" in caplog.text + + +def test_start_container_invalid_config_path( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `start_container` function with a nonexistent config path. + This should log an error and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + config_file = "nonexistent.config" + server_server_config["container"]["image"] = "start_container.image" + server_server_config["container"]["config"] = config_file + + # Create the config path so we ensure it exists + image_path = f"{server_testing_dir}/{server_server_config['container']['image']}" + with open(image_path, "w"): + pass + + assert start_container(ServerConfig(server_server_config)) is None + assert f"Unable to find config file at {os.path.join(server_testing_dir, config_file)}" in caplog.text + + +def test_start_container_valid_paths(mocker: "Fixture", server_server_config: Dict[str, Dict[str, str]]): # noqa: F821 + """ + Test the `start_container` function with valid image and config paths. + This should return a subprocess. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + expected_return = "fake subprocess" + mocker.patch("subprocess.Popen", return_value=expected_return) + mocker.patch("os.path.exists", return_value=True) + assert start_container(ServerConfig(server_server_config)) == expected_return + + +def test_server_started_no_redis_start(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `server_started` function with redis not starting. This should log errors and + return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mock_process = mocker.Mock() + mock_process.stdout = mocker.Mock() + + expected_redis_out_msg = "Reached end of redis output without seeing 'Ready to accept connections'" + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(False, expected_redis_out_msg)) + + assert not server_started(mock_process, "unecessary_config") + assert "Redis is unable to start" in caplog.text + assert 'Check to see if there is an unresponsive instance of redis with "ps -e"' in caplog.text + assert expected_redis_out_msg in caplog.text + + +def test_server_started_process_file_dump_fail( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `server_started` function with the dump to the process file failing. + This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mock_process = mocker.Mock() + mock_process.pid = 1234 + mock_process.stdout = mocker.Mock() + + image_pid = 5678 + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid})) + mocker.patch("merlin.server.server_commands.dump_process_file", return_value=False) + + assert not server_started(mock_process, ServerConfig(server_server_config)) + assert "Unable to create process file for container." in caplog.text + + +@pytest.mark.parametrize( + "server_status", + [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, + ], +) +def test_server_started_server_not_running( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], + server_status: ServerStatus, +): + """ + Test the `server_started` function with the server status returning a non-running status. + This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_status: The server status for this test + """ + mock_process = mocker.Mock() + mock_process.pid = 1234 + mock_process.stdout = mocker.Mock() + + image_pid = 5678 + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid})) + mocker.patch("merlin.server.server_commands.dump_process_file", return_value=True) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + + assert not server_started(mock_process, ServerConfig(server_server_config)) + assert "Unable to start merlin server." in caplog.text + + +def test_server_started_no_issues(mocker: "Fixture", server_server_config: Dict[str, Dict[str, str]]): # noqa: F821 + """ + Test the `server_started` function with no issues starting the server. + This should return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mock_process = mocker.Mock() + mock_process.pid = 1234 + mock_process.stdout = mocker.Mock() + + image_pid = 5678 + mocker.patch("merlin.server.server_commands.parse_redis_output", return_value=(True, {"pid": image_pid, "port": 6379})) + mocker.patch("merlin.server.server_commands.dump_process_file", return_value=True) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + + assert server_started(mock_process, ServerConfig(server_server_config)) + + +def test_start_server_no_running_server(mocker: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with no running server. This should return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=False) + assert not start_server() + + +def test_start_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with no running server. This should return False + and log an error. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=None) + assert not start_server() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_start_server_redis_container_startup_fail(mocker: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with the redis container startup failing. This should return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=True) + mocker.patch("merlin.server.server_commands.start_container", return_value=None) + assert not start_server() + + +def test_start_server_server_did_not_start(mocker: "Fixture"): # noqa: F821 + """ + Test the `start_server` function with the server startup failing. This should return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=True) + mocker.patch("merlin.server.server_commands.start_container", return_value=True) + mocker.patch("merlin.server.server_commands.server_started", return_value=False) + assert not start_server() + + +def test_start_server_successful_start( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], + server_redis_conf_file: str, +): + """ + Test the `start_server` function with a successful start. This should return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_redis_conf_file: The path to a dummy redis configuration file + """ + mocker.patch("merlin.server.server_commands.check_for_not_running_server", return_value=True) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.start_container", return_value=True) + mocker.patch("merlin.server.server_commands.server_started", return_value=True) + mocker.patch("merlin.server.server_commands.RedisUsers") + mocker.patch("merlin.server.server_commands.RedisConfig") + mocker.patch("merlin.server.server_commands.AppYaml") + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_path", return_value=server_redis_conf_file) + mocker.patch("os.path.join", return_value=f"{server_testing_dir}/start_server_app.yaml") + + assert start_server() + + +@pytest.mark.parametrize( + "server_status", + [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, + ], +) +def test_stop_server_server_not_running( + mocker: "Fixture", caplog: "Fixture", server_status: ServerStatus # noqa: F821 # noqa: F821 +): + """ + Test the `stop_server` function with a server that's not running. This should log two messages + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + assert not stop_server() + assert "There is no instance of merlin server running." in caplog.text + assert "Start a merlin server first with 'merlin server start'" in caplog.text + + +def test_stop_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `stop_server` function with no server config being pulled. This should log a message + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=None) + assert not stop_server() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_stop_server_empty_stdout( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: ServerConfig, +): + """ + Test the `stop_server` function with no server config being pulled. This should log a message + and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.pull_process_file", return_value={"parent_pid": 123}) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = b"" + assert not stop_server() + assert "Unable to get the PID for the current merlin server." in caplog.text + + +def test_stop_server_unable_to_stop_server( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: ServerConfig, +): + """ + Test the `stop_server` function with the server status still RUNNING after trying + to kill the server. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.pull_process_file", return_value={"parent_pid": 123}) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = b"some output from status check" + assert not stop_server() + assert "Unable to kill process." in caplog.text + + +def test_stop_server_stop_command_is_not_kill( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_server_config: ServerConfig, +): + """ + Test the `stop_server` function with a stop command that's not 'kill'. + This should run through the command successfully and return True. The subprocess + should run the command we provide in this test. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch( + "merlin.server.server_commands.get_server_status", side_effect=[ServerStatus.RUNNING, ServerStatus.NOT_RUNNING] + ) + mocker.patch("merlin.server.server_commands.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_commands.pull_process_file", return_value={"parent_pid": 123}) + custom_stop_command = "not a kill command" + mocker.patch("merlin.server.server_util.ContainerFormatConfig.get_stop_command", return_value=custom_stop_command) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = b"some output from status check" + assert stop_server() + mock_run.assert_called_with(custom_stop_command.split(), stdout=subprocess.PIPE) + + +@pytest.mark.parametrize( + "server_status", + [ + ServerStatus.NOT_RUNNING, + ServerStatus.MISSING_CONTAINER, + ServerStatus.NOT_INITIALIZED, + ], +) +def test_restart_server_server_not_running(mocker: "Fixture", caplog: "Fixture", server_status: ServerStatus): # noqa: F821 + """ + Test the `restart_server` function with a server that's not running. + This should log two messages and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_status: The server status for this test + """ + caplog.set_level(logging.INFO) + mocker.patch("merlin.server.server_commands.get_server_status", return_value=server_status) + assert not restart_server() + assert "Merlin server is not currently running." in caplog.text + assert "Please start a merlin server instance first with 'merlin server start'" in caplog.text + + +def test_restart_server_successful_restart(mocker: "Fixture"): # noqa: F821 + """ + Test the `restart_server` function with a successful restart. This should call + `stop_server` and `start_server`, and return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + """ + mocker.patch("merlin.server.server_commands.get_server_status", return_value=ServerStatus.RUNNING) + stop_server_mock = mocker.patch("merlin.server.server_commands.stop_server") + start_server_mock = mocker.patch("merlin.server.server_commands.start_server") + assert restart_server() + stop_server_mock.assert_called_once() + start_server_mock.assert_called_once() diff --git a/tests/unit/server/test_server_config.py b/tests/unit/server/test_server_config.py new file mode 100644 index 000000000..51b74a842 --- /dev/null +++ b/tests/unit/server/test_server_config.py @@ -0,0 +1,858 @@ +""" +Tests for the `server_config.py` module. +""" + +import io +import logging +import os +import string +from typing import Dict, Tuple, Union + +import pytest +import yaml + +from merlin.server.server_config import ( + MERLIN_CONFIG_DIR, + PASSWORD_LENGTH, + ServerStatus, + check_process_file_format, + config_merlin_server, + copy_container_command_files, + create_server_config, + dump_process_file, + generate_password, + get_server_status, + parse_redis_output, + pull_process_file, + pull_server_config, + pull_server_image, +) +from merlin.server.server_util import CONTAINER_TYPES, MERLIN_SERVER_SUBDIR, ServerConfig + + +try: + from importlib import resources +except ImportError: + import importlib_resources as resources + + +def test_generate_password_no_pass_command(): + """ + Test the `generate_password` function with no password command. + This should generate a password of 256 (PASSWORD_LENGTH) random ASCII characters. + """ + generated_password = generate_password(PASSWORD_LENGTH) + assert len(generated_password) == PASSWORD_LENGTH + valid_ascii_chars = string.ascii_letters + string.digits + "!@#$%^&*()" + for ch in generated_password: + assert ch in valid_ascii_chars + + +def test_generate_password_with_pass_command(): + """ + Test the `generate_password` function with no password command. + This should generate a password of 256 (PASSWORD_LENGTH) random ASCII characters. + """ + test_pass = "test-password" + generated_password = generate_password(0, pass_command=f"echo {test_pass}") + assert generated_password == test_pass + + +@pytest.mark.parametrize( + "line, expected_return", + [ + (None, (False, "None passed as redis output")), + (b"", (False, "Reached end of redis output without seeing 'Ready to accept connections'")), + (b"Ready to accept connections", (True, {})), + (b"aborting", (False, "aborting")), + (b"Fatal error", (False, "Fatal error")), + ], +) +def test_parse_redis_output_with_basic_input(line: Union[None, bytes], expected_return: Tuple[bool, Union[str, Dict]]): + """ + Test the `parse_redis_output` function with basic input. + Here "basic input" means single line input or None as input. + + :param line: The value to pass in as input to `parse_redis_output` + :param expected_return: The expected return value based on what was passed in for `line` + """ + if line is None: + reader_input = None + else: + buffer = io.BytesIO(line) + reader_input = io.BufferedReader(buffer) + actual_return = parse_redis_output(reader_input) + assert expected_return == actual_return + + +@pytest.mark.parametrize( + "lines, expected_config", + [ + ( # Testing setting vars before initialized message + b"port=6379 blah blah server=127.0.0.1\nServer initialized\nReady to accept connections", + {"port": "6379", "server": "127.0.0.1"}, + ), + ( # Testing setting vars after initialized message + b"Server initialized\nport=6379 blah blah server=127.0.0.1\nReady to accept connections", + {}, + ), + ( # Testing setting vars before + after initialized message + b"blah blah max_connections=100 blah\n" + b"Server initialized\n" + b"port=6379 blah blah server=127.0.0.1\n" + b"Ready to accept connections", + {"max_connections": "100"}, + ), + ], +) +def test_parse_redis_output_with_vars(lines: bytes, expected_config: Tuple[bool, Union[str, Dict]]): + """ + Test the `parse_redis_output` function with input that has variables in lines. + This should set any variable given before the "Server initialized" message is provided. + + We'll test setting vars before the initialized message, after, and both before and after. + + :param lines: The lines to pass in as input to `parse_redis_output` + :param expected_config: The expected config dict based on what was passed in for `lines` + """ + buffer = io.BytesIO(lines) + reader_input = io.BufferedReader(buffer) + _, actual_vars = parse_redis_output(reader_input) + assert expected_config == actual_vars + + +def test_copy_container_command_files_with_existing_files( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Test the `copy_container_command_files` function with files that already exist. + This should skip trying to create the files, log 3 "file already exists" messages, + and return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + caplog.set_level(logging.INFO) + mocker.patch("os.path.exists", return_value=True) + assert copy_container_command_files(server_testing_dir) + file_names = [f"{container}.yaml" for container in CONTAINER_TYPES] + for file in file_names: + assert f"{file} already exists." in caplog.text + + +def test_copy_container_command_files_with_nonexisting_files( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Test the `copy_container_command_files` function with files that don't already exist. + This should create the files, log messages for each file, and return True + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + caplog.set_level(logging.INFO) + + # Mock the os.path.exists function so it returns False + mocker.patch("os.path.exists", return_value=False) + + # Mock the resources.path context manager + mock_path = mocker.patch("merlin.server.server_config.resources.path") + mock_path.return_value.__enter__.return_value = "mocked_file_path" + + # Mock the open builtin + mock_data = mocker.mock_open(read_data="mocked data") + mocker.patch("builtins.open", mock_data) + + assert copy_container_command_files(server_testing_dir) + file_names = [f"{container}.yaml" for container in CONTAINER_TYPES] + for file in file_names: + assert f"Copying file {file} to configuration directory." in caplog.text + + +def test_copy_container_command_files_with_oserror( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Test the `copy_container_command_files` function with an OSError being raised. + This should log an error message and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + # Mock the open function to raise an OSError + mocker.patch("builtins.open", side_effect=OSError("File not writeable")) + + assert not copy_container_command_files(server_testing_dir) + assert f"Destination location {server_testing_dir} is not writable." in caplog.text + + +def test_create_server_config_merlin_config_dir_nonexistent( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Tests the `create_server_config` function with MERLIN_CONFIG_DIR not existing. + This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + nonexistent_dir = f"{server_testing_dir}/merlin_config_dir" + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", nonexistent_dir) + assert not create_server_config() + assert f"Unable to find main merlin configuration directory at {nonexistent_dir}" in caplog.text + + +def test_create_server_config_server_subdir_nonexistent_oserror( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Tests the `create_server_config` function with MERLIN_CONFIG_DIR/MERLIN_SERVER_SUBDIR + not existing and an OSError being raised. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + + # Mock MERLIN_CONFIG_DIR and MERLIN_SERVER_SUBDIR + nonexistent_server_subdir = "test_create_server_config_server_subdir_nonexistent" + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) + mocker.patch("merlin.server.server_config.MERLIN_SERVER_SUBDIR", nonexistent_server_subdir) + + # Mock os.mkdir so it raises an OSError + err_msg = "File not writeable" + mocker.patch("os.mkdir", side_effect=OSError(err_msg)) + assert not create_server_config() + assert err_msg in caplog.text + + +def test_create_server_config_no_server_config( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, +): + """ + Tests the `create_server_config` function with the call to `pull_server_config()` + returning None. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + """ + + # Mock the necessary variables/functions to get us to the pull_server_config call + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) + mocker.patch("merlin.server.server_config.copy_container_command_files", return_value=True) + mock_open_func = mocker.mock_open(read_data="key: value") + mocker.patch("builtins.open", mock_open_func) + + # Mock the pull_server_config call (what we're actually testing) and run the test + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + assert not create_server_config() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_create_server_config_no_server_dir( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, str], +): + """ + Tests the `create_server_config` function with the call to + `server_config.container.get_config_dir()` returning a non-existent path. This should + log a message and create the directory, then return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Mock the necessary variables/functions to get us to the get_config_dir call + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) + mocker.patch("merlin.server.server_config.copy_container_command_files", return_value=True) + mock_open_func = mocker.mock_open(read_data="key: value") + mocker.patch("builtins.open", mock_open_func) + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + + # Mock the get_config_dir call to return a directory that doesn't exist yet + nonexistent_dir = f"{server_testing_dir}/merlin_server" + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_dir", return_value=nonexistent_dir) + + assert create_server_config() + assert os.path.exists(nonexistent_dir) + assert "Creating merlin server directory." in caplog.text + + +def test_config_merlin_server_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `config_merlin_server` function with the call to `pull_server_config()` + returning None. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + assert not config_merlin_server() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def test_config_merlin_server_pass_user_exist( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, str], +): + """ + Tests the `config_merlin_server` function with a password file and user file already + existing. This should log 2 messages and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Create the password file and user file + pass_file = f"{server_testing_dir}/existent_pass_file.txt" + user_file = f"{server_testing_dir}/existent_user_file.txt" + with open(pass_file, "w"), open(user_file, "w"): + pass + + # Mock necessary calls + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_pass_file_path", return_value=pass_file) + mocker.patch("merlin.server.server_util.ContainerConfig.get_user_file_path", return_value=user_file) + + assert config_merlin_server() is None + assert "Password file already exists. Skipping password generation step." in caplog.text + assert "User file already exists." in caplog.text + + +def test_config_merlin_server_pass_user_dont_exist( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, str], +): + """ + Tests the `config_merlin_server` function with a password file and user file that don't + already exist. This should log 2 messages, create the files, and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Create vars for the nonexistent password file and user file + pass_file = f"{server_testing_dir}/nonexistent_pass_file.txt" + user_file = f"{server_testing_dir}/nonexistent_user_file.txt" + + # Mock necessary calls + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_pass_file_path", return_value=pass_file) + mocker.patch("merlin.server.server_util.ContainerConfig.get_user_file_path", return_value=user_file) + + assert config_merlin_server() is None + assert os.path.exists(pass_file) + assert os.path.exists(user_file) + assert "Creating password file for merlin server container." in caplog.text + assert f"User {os.environ.get('USER')} created in user file for merlin server container" in caplog.text + + +def setup_pull_server_config_mock( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_server_config: Dict[str, Dict[str, str]], +): + """ + Setup the necessary mocker calls for the `pull_server_config` function. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_util.AppYaml.get_data", return_value=server_app_yaml_contents) + mocker.patch("merlin.server.server_config.MERLIN_CONFIG_DIR", server_testing_dir) + mock_data = mocker.mock_open(read_data=str(server_server_config)) + mocker.patch("builtins.open", mock_data) + + +@pytest.mark.parametrize( + "key_to_delete, expected_log_message", + [ + ("container", 'Unable to find "container" object in {default_app_yaml}'), + ("container.format", 'Unable to find "format" in {default_app_yaml}'), + ("process", "Process config not found in {default_app_yaml}"), + ], +) +def test_pull_server_config_missing_config_keys( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_server_config: Dict[str, Dict[str, str]], + key_to_delete: str, + expected_log_message: str, +): + """ + Test the `pull_server_config` function with missing container-related keys in the + app.yaml file contents. This should log an error message and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param key_to_delete: The key to delete from the app.yaml contents + :param expected_log_message: The expected log message when the key is missing + """ + # Handle nested key deletion + keys = key_to_delete.split(".") + temp_app_yaml = server_app_yaml_contents + for key in keys[:-1]: + temp_app_yaml = temp_app_yaml[key] + del temp_app_yaml[keys[-1]] + + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + + assert pull_server_config() is None + default_app_yaml = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") + assert expected_log_message.format(default_app_yaml=default_app_yaml) in caplog.text + + +@pytest.mark.parametrize("key_to_delete", ["command", "run_command", "stop_command", "pull_command"]) +def test_pull_server_config_missing_format_needed_keys( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_container_format_config_data: Dict[str, str], + server_server_config: Dict[str, Dict[str, str]], + key_to_delete: str, +): + """ + Test the `pull_server_config` function with necessary format keys missing in the + singularity.yaml file contents. This should log an error message and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param key_to_delete: The key to delete from the singularity.yaml contents + """ + del server_container_format_config_data[key_to_delete] + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + + assert pull_server_config() is None + format_file_basename = server_app_yaml_contents["container"]["format"] + ".yaml" + format_file = os.path.join(server_testing_dir, MERLIN_SERVER_SUBDIR) + format_file = os.path.join(format_file, format_file_basename) + assert f'Unable to find necessary "{key_to_delete}" value in format config file {format_file}' in caplog.text + + +@pytest.mark.parametrize("key_to_delete", ["status", "kill"]) +def test_pull_server_config_missing_process_needed_key( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_process_config_data: Dict[str, str], + server_server_config: Dict[str, Dict[str, str]], + key_to_delete: str, +): + """ + Test the `pull_server_config` function with necessary process keys missing. + This should log an error message and return None. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param key_to_delete: The key to delete from the process config entry + """ + del server_process_config_data[key_to_delete] + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + + assert pull_server_config() is None + default_app_yaml = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") + assert f'Process necessary "{key_to_delete}" command configuration not found in {default_app_yaml}' in caplog.text + + +def test_pull_server_config_no_issues( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_app_yaml_contents: Dict[str, Union[str, int]], + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_config` function without any problems. This should + return a ServerConfig object. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_app_yaml_contents: A dict of app.yaml configurations + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + setup_pull_server_config_mock(mocker, server_testing_dir, server_app_yaml_contents, server_server_config) + assert isinstance(pull_server_config(), ServerConfig) + + +def test_pull_server_image_no_server_config(mocker: "Fixture", caplog: "Fixture"): # noqa: F821 + """ + Test the `pull_server_image` function with no server config being found. + This should return False and log an error message. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + """ + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + assert not pull_server_image() + assert 'Try to run "merlin server init" again to reinitialize values.' in caplog.text + + +def setup_pull_server_image_mock( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], + config_dir: str, + config_file: str, + image_file: str, + create_config_file: bool = False, + create_image_file: bool = False, +): + """ + Set up the necessary mock calls for the `pull_server_image` function. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + image_url = "docker://redis" + image_path = f"{server_testing_dir}/{image_file}" + + os.makedirs(config_dir, exist_ok=True) + + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_dir", return_value=config_dir) + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_name", return_value=config_file) + mocker.patch("merlin.server.server_util.ContainerConfig.get_image_url", return_value=image_url) + mocker.patch("merlin.server.server_util.ContainerConfig.get_image_path", return_value=image_path) + + if create_config_file: + with open(os.path.join(config_dir, config_file), "w"): + pass + + if create_image_file: + with open(image_path, "w"): + pass + + +def test_pull_server_image_no_image_path_no_config_path( + mocker: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_image` function with no image path and no configuration + path. This should run a subprocess for the image path and create the configuration + file. It should also return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + # Set up mock calls to simulate the setup of this function + config_dir = f"{server_testing_dir}/config_dir" + config_file = "redis.conf" + image_file = "pull_server_image_no_image_path_no_config_path_image_nonexistent.sif" + setup_pull_server_image_mock(mocker, server_testing_dir, server_server_config, config_dir, config_file, image_file) + mocked_subprocess = mocker.patch("subprocess.run") + + # Mock the open function + read_data = "Mocked file content" + mocked_open = mocker.mock_open(read_data=read_data) + mocked_open.write = mocker.Mock() + mocker.patch("builtins.open", mocked_open) + + # Call the function + assert pull_server_image() + + # Assert that the subprocess call to pull the image was called + mocked_subprocess.assert_called_once() + + # Assert that open was called with the correct arguments + mocked_open.assert_any_call(os.path.join(config_dir, config_file), "w") + with resources.path("merlin.server", config_file) as file: + mocked_open.assert_any_call(file, "r") + assert mocked_open.call_count == 2 + + # Assert that the write method was called with the expected content + mocked_open().write.assert_called_once_with(read_data) + + +def test_pull_server_image_both_paths_exist( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_image` function with both an image path and a configuration + path that both exist. This should log two messages and return True. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + caplog.set_level(logging.INFO) + + # Set up mock calls to simulate the setup of this function + config_dir = f"{server_testing_dir}/config_dir" + config_file = "pull_server_image_both_paths_exist_config.yaml" + image_file = "pull_server_image_both_paths_exist_image.sif" + setup_pull_server_image_mock( + mocker, + server_testing_dir, + server_server_config, + config_dir, + config_file, + image_file, + create_config_file=True, + create_image_file=True, + ) + + assert pull_server_image() + assert f"{image_file} already exists." in caplog.text + assert "Redis configuration file already exist." in caplog.text + + +def test_pull_server_image_os_error( + mocker: "Fixture", # noqa: F821 + caplog: "Fixture", # noqa: F821 + server_testing_dir: str, + server_server_config: Dict[str, Dict[str, str]], +): + """ + Test the `pull_server_image` function with an image path but no configuration + path. We'll force this to raise an OSError when writing to the configuration file + to ensure it's handled properly. This should log an error and return False. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param caplog: A built-in fixture from the pytest library to capture logs + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + # Set up mock calls to simulate the setup of this function + config_dir = f"{server_testing_dir}/config_dir" + config_file = "pull_server_image_os_error_config.yaml" + image_file = "pull_server_image_os_error_config_nonexistent.sif" + setup_pull_server_image_mock( + mocker, + server_testing_dir, + server_server_config, + config_dir, + config_file, + image_file, + create_image_file=True, + ) + + # Mock the open function + mocker.patch("builtins.open", side_effect=OSError) + + # Run the test + assert not pull_server_image() + assert f"Destination location {config_dir} is not writable." in caplog.text + + +@pytest.mark.parametrize( + "server_config_exists, config_exists, image_exists, pfile_exists, expected_status", + [ + (False, True, True, True, ServerStatus.NOT_INITIALIZED), # No server config + (True, False, True, True, ServerStatus.NOT_INITIALIZED), # Config dir does not exist + (True, True, False, True, ServerStatus.MISSING_CONTAINER), # Image path does not exist + (True, True, True, False, ServerStatus.NOT_RUNNING), # Pfile path does not exist + ], +) +def test_get_server_status_initial_checks( + mocker: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], + server_config_exists: bool, + config_exists: bool, + image_exists: bool, + pfile_exists: bool, + expected_status: ServerStatus, +): + """ + Test the `get_server_status` function for the initial conditional checks that it looks for. + These checks include: + - no server configuration -> should return NOT_INITIALIZED + - no config directory path -> should return NOT_INITIALIZED + - no image path -> should return MISSING_CONTAINER + - no password file -> should return NOT_RUNNING + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + :param server_config_exists: A boolean to denote whether the server config exists in this test or not + :param config_exists: A boolean to denote whether the config dir exists in this test or not + :param image_exists: A boolean to denote whether the image path exists in this test or not + :param pfile_exists: A boolean to denote whether the password file exists in this test or not + :param expected_status: The status we're expecting `get_server_status` to return for this test + """ + # Mock the necessary calls + if server_config_exists: + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("merlin.server.server_util.ContainerConfig.get_config_dir", return_value="config_dir") + mocker.patch("merlin.server.server_util.ContainerConfig.get_image_path", return_value="image_path") + mocker.patch("merlin.server.server_util.ContainerConfig.get_pfile_path", return_value="pfile_path") + + # Mock os.path.exists to return the desired values + mocker.patch( + "os.path.exists", + side_effect=lambda path: {"config_dir": config_exists, "image_path": image_exists, "pfile_path": pfile_exists}.get( + path, False + ), + ) + else: + mocker.patch("merlin.server.server_config.pull_server_config", return_value=None) + + # Call the function and assert the expected status + assert get_server_status() == expected_status + + +@pytest.mark.parametrize( + "stdout_val, expected_status", + [ + (b"", ServerStatus.NOT_RUNNING), # No stdout from subprocess + (b"Successfully started", ServerStatus.RUNNING), # Stdout from subprocess exists + ], +) +def test_get_server_status_subprocess_check( + mocker: "Fixture", # noqa: F821 + server_server_config: Dict[str, Dict[str, str]], + stdout_val: bytes, + expected_status: ServerStatus, +): + """ + Test the `get_server_status` function with empty stdout return from the subprocess run. + This should return a NOT_RUNNING status. + + :param mocker: A built-in fixture from the pytest-mock library to create a Mock object + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + mocker.patch("merlin.server.server_config.pull_server_config", return_value=ServerConfig(server_server_config)) + mocker.patch("os.path.exists", return_value=True) + mocker.patch("merlin.server.server_config.pull_process_file", return_value={"parent_pid": 123}) + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = stdout_val + + assert get_server_status() == expected_status + + +@pytest.mark.parametrize( + "data_to_test, expected_result", + [ + ({"image_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No parent_pid entry + ({"parent_pid": 123, "port": 6379, "hostname": "dummy_server"}, False), # No image_pid entry + ({"parent_pid": 123, "image_pid": 456, "hostname": "dummy_server"}, False), # No port entry + ({"parent_pid": 123, "image_pid": 123, "port": 6379}, False), # No hostname entry + ({"parent_pid": 123, "image_pid": 123, "port": 6379, "hostname": "dummy_server"}, True), # All required entries exist + ], +) +def test_check_process_file_format(data_to_test: Dict[str, Union[int, str]], expected_result: bool): + """ + Test the `check_process_file_format` function. The first 4 parametrized tests above should all + return False as they're all missing a required key. The final parametrized test above should return + True since it has every required key. + + :param data_to_test: The data dict that we'll pass in to the `check_process_file_format` function + :param expected_result: The return value we expect based on `data_to_test` + """ + assert check_process_file_format(data_to_test) == expected_result + + +def test_pull_process_file_valid_file(server_testing_dir: str, server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `pull_process_file` function with a valid process file. This test will create a test + process file with valid contents that `pull_process_file` will read in and return. + + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_process_file_contents: A fixture representing process file contents + """ + # Create the valid process file in our temp testing directory + process_filepath = f"{server_testing_dir}/valid_process_file.yaml" + with open(process_filepath, "w") as process_file: + yaml.dump(server_process_file_contents, process_file) + + # Run the test + assert pull_process_file(process_filepath) == server_process_file_contents + + +def test_pull_process_file_invalid_file(server_testing_dir: str, server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `pull_process_file` function with an invalid process file. This test will create a test + process file with invalid contents that `pull_process_file` will try to read in. Once it sees + that the file is invalid it will return None. + + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_process_file_contents: A fixture representing process file contents + """ + # Remove a key from the process file contents so that it's no longer valid + del server_process_file_contents["hostname"] + + # Create the invalid process file in our temp testing directory + process_filepath = f"{server_testing_dir}/invalid_process_file.yaml" + with open(process_filepath, "w") as process_file: + yaml.dump(server_process_file_contents, process_file) + + # Run the test + assert pull_process_file(process_filepath) is None + + +def test_dump_process_file_invalid_file(server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `dump_process_file` function with invalid process file data. This should return False. + + :param server_process_file_contents: A fixture representing process file contents + """ + # Remove a key from the process file contents so that it's no longer valid and run the test + del server_process_file_contents["parent_pid"] + assert not dump_process_file(server_process_file_contents, "some_filepath.yaml") + + +def test_dump_process_file_valid_file(server_testing_dir: str, server_process_file_contents: Dict[str, Union[int, str]]): + """ + Test the `dump_process_file` function with invalid process file data. This should return False. + + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_process_file_contents: A fixture representing process file contents + """ + process_filepath = f"{server_testing_dir}/dumped_process_file.yaml" + assert dump_process_file(server_process_file_contents, process_filepath) + assert os.path.exists(process_filepath) diff --git a/tests/unit/server/test_server_util.py b/tests/unit/server/test_server_util.py new file mode 100644 index 000000000..909cb7cdf --- /dev/null +++ b/tests/unit/server/test_server_util.py @@ -0,0 +1,565 @@ +""" +Tests for the `server_util.py` module. +""" + +import filecmp +import hashlib +import os +from typing import Dict, Union + +import pytest + +from merlin.server.server_util import ( + AppYaml, + ContainerConfig, + ContainerFormatConfig, + ProcessConfig, + RedisConfig, + RedisUsers, + ServerConfig, + valid_ipv4, + valid_port, +) + + +@pytest.mark.parametrize( + "valid_ip", + [ + "0.0.0.0", + "127.0.0.1", + "14.105.200.58", + "255.255.255.255", + ], +) +def test_valid_ipv4_valid_ip(valid_ip: str): + """ + Test the `valid_ipv4` function with valid IPs. + This should return True. + + :param valid_ip: A valid port to test. These are pulled from the parametrized + list defined above this test. + """ + assert valid_ipv4(valid_ip) + + +@pytest.mark.parametrize( + "invalid_ip", + [ + "256.0.0.1", + "-1.0.0.1", + None, + "127.0.01", + ], +) +def test_valid_ipv4_invalid_ip(invalid_ip: Union[str, None]): + """ + Test the `valid_ipv4` function with invalid IPs. + An IP is valid if every integer separated by the '.' delimiter are between 0 and 255. + This should return False for both IPs tested here. + + :param invalid_ip: An invalid port to test. These are pulled from the parametrized + list defined above this test. + """ + assert not valid_ipv4(invalid_ip) + + +@pytest.mark.parametrize( + "valid_input", + [ + 1, + 433, + 65535, + ], +) +def test_valid_port_valid_input(valid_input: int): + """ + Test the `valid_port` function with valid port numbers. + Valid ports are ports between 1 and 65535. + This should return True. + + :param valid_input: A valid input value to test. These are pulled from the parametrized + list defined above this test. + """ + assert valid_port(valid_input) + + +@pytest.mark.parametrize( + "invalid_input", + [ + -1, + 0, + 65536, + ], +) +def test_valid_port_invalid_input(invalid_input: int): + """ + Test the `valid_port` function with invalid inputs. + Valid ports are ports between 1 and 65535. + This should return False for each invalid input tested. + + :param invalid_input: An invalid input value to test. These are pulled from the parametrized + list defined above this test. + """ + assert not valid_port(invalid_input) + + +class TestContainerConfig: + """Tests for the ContainerConfig class.""" + + def test_init_with_complete_data(self, server_container_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data. + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + config = ContainerConfig(server_container_config_data) + assert config.format == server_container_config_data["format"] + assert config.image_type == server_container_config_data["image_type"] + assert config.image == server_container_config_data["image"] + assert config.url == server_container_config_data["url"] + assert config.config == server_container_config_data["config"] + assert config.config_dir == server_container_config_data["config_dir"] + assert config.pfile == server_container_config_data["pfile"] + assert config.pass_file == server_container_config_data["pass_file"] + assert config.user_file == server_container_config_data["user_file"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data. + """ + incomplete_data = {"format": "docker"} + config = ContainerConfig(incomplete_data) + assert config.format == incomplete_data["format"] + assert config.image_type == ContainerConfig.IMAGE_TYPE + assert config.image == ContainerConfig.IMAGE_NAME + assert config.url == ContainerConfig.REDIS_URL + assert config.config == ContainerConfig.CONFIG_FILE + assert config.config_dir == ContainerConfig.CONFIG_DIR + assert config.pfile == ContainerConfig.PROCESS_FILE + assert config.pass_file == ContainerConfig.PASSWORD_FILE + assert config.user_file == ContainerConfig.USERS_FILE + + @pytest.mark.parametrize( + "attr_name", + [ + "image", + "config", + "pfile", + "pass_file", + "user_file", + ], + ) + def test_get_path_methods(self, server_container_config_data: Dict[str, str], attr_name: str): + """ + Tests that get_*_path methods construct the correct path. + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param attr_name: Name of the attribute to be tested. These are pulled from the parametrized list defined above this test. + """ + config = ContainerConfig(server_container_config_data) + get_path_method = getattr(config, f"get_{attr_name}_path") # Dynamically get the method based on attr_name + expected_path = os.path.join(server_container_config_data["config_dir"], server_container_config_data[attr_name]) + assert get_path_method() == expected_path + + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_format", "format"), + ("get_image_type", "image_type"), + ("get_image_name", "image"), + ("get_image_url", "url"), + ("get_config_name", "config"), + ("get_config_dir", "config_dir"), + ("get_pfile_name", "pfile"), + ("get_pass_file_name", "pass_file"), + ("get_user_file_name", "user_file"), + ], + ) + def test_getter_methods(self, server_container_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values. + + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ContainerConfig(server_container_config_data) + getter = getattr(config, getter_name) + assert getter() == server_container_config_data[expected_attr] + + def test_get_container_password(self, server_testing_dir: str, server_container_config_data: Dict[str, str]): + """ + Test that the `get_container_password` method is reading the password file properly. + + :param server_testing_dir: The path to the the temp output directory for server tests + :param server_container_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + # Write a fake password to the password file + test_password = "super-secret-password" + temp_pass_file = f"{server_testing_dir}/temp.pass" + with open(temp_pass_file, "w") as pass_file: + pass_file.write(test_password) + + # Use temp pass file + orig_pass_file = server_container_config_data["pass_file"] + server_container_config_data["pass_file"] = temp_pass_file + + try: + # Run the test + config = ContainerConfig(server_container_config_data) + assert config.get_container_password() == test_password + except Exception as exc: + # If there was a problem, reset to the original password file + server_container_config_data["pass_file"] = orig_pass_file + raise exc + + +class TestContainerFormatConfig: + """Tests for the ContainerFormatConfig class.""" + + def test_init_with_complete_data(self, server_container_format_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data. + + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + """ + config = ContainerFormatConfig(server_container_format_config_data) + assert config.command == server_container_format_config_data["command"] + assert config.run_command == server_container_format_config_data["run_command"] + assert config.stop_command == server_container_format_config_data["stop_command"] + assert config.pull_command == server_container_format_config_data["pull_command"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data. + """ + incomplete_data = {"command": "docker"} + config = ContainerFormatConfig(incomplete_data) + assert config.command == incomplete_data["command"] + assert config.run_command == config.RUN_COMMAND + assert config.stop_command == config.STOP_COMMAND + assert config.pull_command == config.PULL_COMMAND + + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_command", "command"), + ("get_run_command", "run_command"), + ("get_stop_command", "stop_command"), + ("get_pull_command", "pull_command"), + ], + ) + def test_getter_methods(self, server_container_format_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values. + + :param server_container_format_config_data: A pytest fixture of test data to pass to the ContainerFormatConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ContainerFormatConfig(server_container_format_config_data) + getter = getattr(config, getter_name) + assert getter() == server_container_format_config_data[expected_attr] + + +class TestProcessConfig: + """Tests for the ProcessConfig class.""" + + def test_init_with_complete_data(self, server_process_config_data: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data. + + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + """ + config = ProcessConfig(server_process_config_data) + assert config.status == server_process_config_data["status"] + assert config.kill == server_process_config_data["kill"] + + def test_init_with_missing_data(self): + """ + Tests that __init__ uses defaults for missing data. + """ + incomplete_data = {"status": "status {pid}"} + config = ProcessConfig(incomplete_data) + assert config.status == incomplete_data["status"] + assert config.kill == config.KILL_COMMAND + + @pytest.mark.parametrize( + "getter_name, expected_attr", + [ + ("get_status_command", "status"), + ("get_kill_command", "kill"), + ], + ) + def test_getter_methods(self, server_process_config_data: Dict[str, str], getter_name: str, expected_attr: str): + """ + Tests that all getter methods return the correct attribute values. + + :param server_process_config_data: A pytest fixture of test data to pass to the ProcessConfig class + :param getter_name: Name of the getter method to test. This is pulled from the parametrized list defined above this test. + :param expected_attr: Name of the corresponding attribute. This is pulled from the parametrized list defined above this test. + """ + config = ProcessConfig(server_process_config_data) + getter = getattr(config, getter_name) + assert getter() == server_process_config_data[expected_attr] + + +class TestServerConfig: + """Tests for the ServerConfig class.""" + + def test_init_with_complete_data(self, server_server_config: Dict[str, str]): + """ + Tests that __init__ populates attributes correctly with complete data. + + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + config = ServerConfig(server_server_config) + assert config.container == ContainerConfig(server_server_config["container"]) + assert config.process == ProcessConfig(server_server_config["process"]) + assert config.container_format == ContainerFormatConfig(server_server_config["singularity"]) + + def test_init_with_missing_data(self, server_process_config_data: Dict[str, str]): + """ + Tests that __init__ uses None for missing data. + + :param server_process_config_data: A pytest fixture of test data to pass to the ContainerConfig class + """ + incomplete_data = {"process": server_process_config_data} + config = ServerConfig(incomplete_data) + assert config.process == ProcessConfig(server_process_config_data) + assert config.container is None + assert config.container_format is None + + +class TestRedisUsers: + """ + Tests for the RedisUsers class. + + TODO add integration test(s) for `apply_to_redis` method of this class. + """ + + class TestUser: + """Tests for the RedisUsers.User class.""" + + def test_initializaiton(self): + """Test the initialization process of the User class.""" + user = RedisUsers.User() + assert user.status == "on" + assert user.hash_password == hashlib.sha256(b"password").hexdigest() + assert user.keys == "*" + assert user.channels == "*" + assert user.commands == "@all" + + def test_parse_dict(self): + """Test the `parse_dict` method of the User class.""" + test_dict = { + "status": "test_status", + "hash_password": "test_password", + "keys": "test_keys", + "channels": "test_channels", + "commands": "test_commands", + } + user = RedisUsers.User() + user.parse_dict(test_dict) + assert user.status == test_dict["status"] + assert user.hash_password == test_dict["hash_password"] + assert user.keys == test_dict["keys"] + assert user.channels == test_dict["channels"] + assert user.commands == test_dict["commands"] + + def test_get_user_dict(self): + """Test the `get_user_dict` method of the User class.""" + test_dict = { + "status": "test_status", + "hash_password": "test_password", + "keys": "test_keys", + "channels": "test_channels", + "commands": "test_commands", + "invalid_key": "invalid_val", + } + user = RedisUsers.User() + user.parse_dict(test_dict) # Set the test values + actual_dict = user.get_user_dict() + assert "invalid_key" not in actual_dict # Check that the invalid key isn't parsed + + # Check that the values are as expected + for key, val in actual_dict.items(): + if key == "status": + assert val == "on" + else: + assert val == test_dict[key] + + def test_set_password(self): + """Test the `set_password` method of the User class.""" + user = RedisUsers.User() + pass_to_set = "dummy_password" + user.set_password(pass_to_set) + assert user.hash_password == hashlib.sha256(bytes(pass_to_set, "utf-8")).hexdigest() + + def test_initialization(self, server_redis_users_file: str, server_users: dict): + """ + Test the initialization process of the RedisUsers class. + + :param server_redis_users_file: The path to a dummy redis users file + :param server_users: A dict of test user configurations + """ + redis_users = RedisUsers(server_redis_users_file) + assert redis_users.filename == server_redis_users_file + assert len(redis_users.users) == len(server_users) + + def test_write(self, server_redis_users_file: str, server_testing_dir: str): + """ + Test that the write functionality works by writing the contents of a dummy + users file to a blank users file. + + :param server_redis_users_file: The path to a dummy redis users file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_redis_users_file = f"{server_testing_dir}/redis_copy.users" + + # Create a RedisUsers object with the basic redis users file + redis_users = RedisUsers(server_redis_users_file) + + # Change the filepath of the redis users file to be the copy that we'll write to + redis_users.filename = copy_redis_users_file + + # Run the test + redis_users.write() + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_redis_users_file, copy_redis_users_file) + + def test_add_user_nonexistent(self, server_redis_users_file: str): + """ + Test the `add_user` method with a user that doesn't exists. + This should return True and add the user to the list of users. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + num_users_before = len(redis_users.users) + assert redis_users.add_user("new_user") + assert len(redis_users.users) == num_users_before + 1 + + def test_add_user_exists(self, server_redis_users_file: str): + """ + Test the `add_user` method with a user that already exists. + This should return False. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.add_user("test_user") + + def test_set_password_valid(self, server_redis_users_file: str): + """ + Test the `set_password` method with a user that exists. + This should return True and change the password for the user. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + pass_to_set = "new_password" + assert redis_users.set_password("test_user", pass_to_set) + expected_hash_pass = hashlib.sha256(bytes(pass_to_set, "utf-8")).hexdigest() + assert redis_users.users["test_user"].hash_password == expected_hash_pass + + def test_set_password_invalid(self, server_redis_users_file: str): + """ + Test the `set_password` method with a user that doesn't exist. + This should return False. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.set_password("nonexistent_user", "new_password") + + def test_remove_user_valid(self, server_redis_users_file: str): + """ + Test the `remove_user` method with a user that exists. + This should return True and remove the user from the list of users. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + num_users_before = len(redis_users.users) + assert redis_users.remove_user("test_user") + assert len(redis_users.users) == num_users_before - 1 + + def test_remove_user_invalid(self, server_redis_users_file: str): + """ + Test the `remove_user` method with a user that doesn't exist. + This should return False and not modify the user list. + + :param server_redis_users_file: The path to a dummy redis users file + """ + redis_users = RedisUsers(server_redis_users_file) + assert not redis_users.remove_user("nonexistent_user") + + +class TestAppYaml: + """Tests for the AppYaml class.""" + + def test_initialization(self, server_app_yaml: str, server_app_yaml_contents: dict): + """ + Test the initialization process of the AppYaml class. + + :param server_app_yaml: The path to an app.yaml file + :param server_app_yaml_contents: A dict of app.yaml configurations + """ + app_yaml = AppYaml(server_app_yaml) + assert app_yaml.get_data() == server_app_yaml_contents + + def test_apply_server_config(self, server_app_yaml: str, server_server_config: Dict[str, str]): + """ + Test the `apply_server_config` method. This should update the data attribute. + + :param server_app_yaml: The path to an app.yaml file + :param server_server_config: A pytest fixture of test data to pass to the ServerConfig class + """ + app_yaml = AppYaml(server_app_yaml) + server_config = ServerConfig(server_server_config) + redis_config = RedisConfig(server_config.container.get_config_path()) + app_yaml.apply_server_config(server_config) + + assert app_yaml.data[app_yaml.broker_name]["name"] == server_config.container.get_image_type() + assert app_yaml.data[app_yaml.broker_name]["username"] == "default" + assert app_yaml.data[app_yaml.broker_name]["password"] == server_config.container.get_pass_file_path() + assert app_yaml.data[app_yaml.broker_name]["server"] == redis_config.get_ip_address() + assert app_yaml.data[app_yaml.broker_name]["port"] == redis_config.get_port() + + assert app_yaml.data[app_yaml.results_name]["name"] == server_config.container.get_image_type() + assert app_yaml.data[app_yaml.results_name]["username"] == "default" + assert app_yaml.data[app_yaml.results_name]["password"] == server_config.container.get_pass_file_path() + assert app_yaml.data[app_yaml.results_name]["server"] == redis_config.get_ip_address() + assert app_yaml.data[app_yaml.results_name]["port"] == redis_config.get_port() + + def test_update_data(self, server_app_yaml: str): + """ + Test the `update_data` method. This should update the data attribute. + + :param server_app_yaml: The path to an app.yaml file + """ + app_yaml = AppYaml(server_app_yaml) + new_data = {app_yaml.broker_name: {"username": "new_user"}} + app_yaml.update_data(new_data) + + assert app_yaml.data[app_yaml.broker_name]["username"] == "new_user" + + def test_write(self, server_app_yaml: str, server_testing_dir: str): + """ + Test the `write` method. This should write data to a file. + + :param server_app_yaml: The path to an app.yaml file + :param server_testing_dir: The path to the the temp output directory for server tests + """ + copy_app_yaml = f"{server_testing_dir}/app_copy.yaml" + + # Create a AppYaml object with the basic app.yaml file + app_yaml = AppYaml(server_app_yaml) + + # Run the test + app_yaml.write(copy_app_yaml) + + # Check that the contents of the copied file match the contents of the basic file + assert filecmp.cmp(server_app_yaml, copy_app_yaml) diff --git a/tests/unit/study/__init__.py b/tests/unit/study/__init__.py index 57477ea1f..37cabcad1 100644 --- a/tests/unit/study/__init__.py +++ b/tests/unit/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/study/status_test_files/combine_status_files.py b/tests/unit/study/status_test_files/combine_status_files.py index f7021a97a..b52b35f1d 100644 --- a/tests/unit/study/status_test_files/combine_status_files.py +++ b/tests/unit/study/status_test_files/combine_status_files.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/study/status_test_files/shared_tests.py b/tests/unit/study/status_test_files/shared_tests.py index fb31b96a7..3e6b0fde8 100644 --- a/tests/unit/study/status_test_files/shared_tests.py +++ b/tests/unit/study/status_test_files/shared_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/study/status_test_files/status_test_variables.py b/tests/unit/study/status_test_files/status_test_variables.py index ffb3cba31..6a7f7eb28 100644 --- a/tests/unit/study/status_test_files/status_test_variables.py +++ b/tests/unit/study/status_test_files/status_test_variables.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/study/test_detailed_status.py b/tests/unit/study/test_detailed_status.py index ae278c975..e6eea4748 100644 --- a/tests/unit/study/test_detailed_status.py +++ b/tests/unit/study/test_detailed_status.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/study/test_status.py b/tests/unit/study/test_status.py index 9d602848f..6786a1230 100644 --- a/tests/unit/study/test_status.py +++ b/tests/unit/study/test_status.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.12.2b1. +# This file is part of Merlin, Version: 1.12.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/unit/test_examples_generator.py b/tests/unit/test_examples_generator.py new file mode 100644 index 000000000..7d4d879fb --- /dev/null +++ b/tests/unit/test_examples_generator.py @@ -0,0 +1,475 @@ +""" +Tests for the `merlin/examples/generator.py` module. +""" + +import os +from typing import List + +import pytest +from tabulate import tabulate + +from merlin.examples.generator import ( + EXAMPLES_DIR, + gather_all_examples, + gather_example_dirs, + list_examples, + setup_example, + write_example, +) +from tests.utils import create_dir + + +EXAMPLES_GENERATOR_DIR = "{temp_output_dir}/examples_generator" + + +def test_gather_example_dirs(): + """Test the `gather_example_dirs` function.""" + example_workflows = [ + "feature_demo", + "flux", + "hello", + "hpc_demo", + "iterative_demo", + "lsf", + "null_spec", + "openfoam_wf", + "openfoam_wf_no_docker", + "openfoam_wf_singularity", + "optimization", + "remote_feature_demo", + "restart", + "restart_delay", + "simple_chain", + "slurm", + ] + expected = {} + for wf_dir in example_workflows: + expected[wf_dir] = wf_dir + actual = gather_example_dirs() + assert actual == expected + + +def test_gather_all_examples(): + """Test the `gather_all_examples` function.""" + expected = [ + f"{EXAMPLES_DIR}/feature_demo/feature_demo.yaml", + f"{EXAMPLES_DIR}/flux/flux_local.yaml", + f"{EXAMPLES_DIR}/flux/flux_par_restart.yaml", + f"{EXAMPLES_DIR}/flux/flux_par.yaml", + f"{EXAMPLES_DIR}/flux/paper.yaml", + f"{EXAMPLES_DIR}/hello/hello_samples.yaml", + f"{EXAMPLES_DIR}/hello/hello.yaml", + f"{EXAMPLES_DIR}/hello/my_hello.yaml", + f"{EXAMPLES_DIR}/hpc_demo/hpc_demo.yaml", + f"{EXAMPLES_DIR}/iterative_demo/iterative_demo.yaml", + f"{EXAMPLES_DIR}/lsf/lsf_par_srun.yaml", + f"{EXAMPLES_DIR}/lsf/lsf_par.yaml", + f"{EXAMPLES_DIR}/null_spec/null_chain.yaml", + f"{EXAMPLES_DIR}/null_spec/null_spec.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf_docker_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf/openfoam_wf.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity.yaml", + f"{EXAMPLES_DIR}/openfoam_wf_singularity/openfoam_wf_singularity_template.yaml", + f"{EXAMPLES_DIR}/optimization/optimization_basic.yaml", + f"{EXAMPLES_DIR}/remote_feature_demo/remote_feature_demo.yaml", + f"{EXAMPLES_DIR}/restart/restart.yaml", + f"{EXAMPLES_DIR}/restart_delay/restart_delay.yaml", + f"{EXAMPLES_DIR}/simple_chain/simple_chain.yaml", + f"{EXAMPLES_DIR}/slurm/slurm_par_restart.yaml", + f"{EXAMPLES_DIR}/slurm/slurm_par.yaml", + ] + actual = gather_all_examples() + assert sorted(actual) == sorted(expected) + + +def test_write_example_dir(examples_testing_dir: str): + """ + Test the `write_example` function with the src_path as a directory. + + :param examples_testing_dir: The path to the the temp output directory for examples tests + """ + dir_to_copy = f"{EXAMPLES_DIR}/feature_demo/" + dst_dir = f"{examples_testing_dir}/write_example_dir" + write_example(dir_to_copy, dst_dir) + assert sorted(os.listdir(dir_to_copy)) == sorted(os.listdir(dst_dir)) + + +def test_write_example_file(examples_testing_dir: str): + """ + Test the `write_example` function with the src_path as a file. + + :param examples_testing_dir: The path to the the temp output directory for examples tests + """ + file_to_copy = f"{EXAMPLES_DIR}/flux/flux_par.yaml" + dst_path = f"{examples_testing_dir}/flux_par.yaml" + write_example(file_to_copy, dst_path) + assert os.path.exists(dst_path) + + +def test_list_examples(): + """Test the `list_examples` function to see if it gives us all of the examples that we want.""" + expected_headers = ["name", "description"] + expected_rows = [ + ["feature_demo", "Run 10 hello worlds."], + ["flux_local", "Run a scan through Merlin/Maestro"], + ["flux_par", "A simple ensemble of parallel MPI jobs run by flux."], + ["flux_par_restart", "A simple ensemble of parallel MPI jobs run by flux."], + ["paper_flux", "Use flux to run single core MPI jobs and record timings."], + ["hello", "a very simple merlin workflow"], + ["hello_samples", "a very simple merlin workflow, with samples"], + ["hpc_demo", "Demo running a workflow on HPC machines"], + ["iterative_demo", "Demo of a workflow with self driven iteration/looping"], + ["lsf_par", "A simple ensemble of parallel MPI jobs run by lsf (jsrun)."], + ["lsf_par_srun", "A simple ensemble of parallel MPI jobs run by lsf using the srun wrapper (srun)."], + [ + "null_chain", + "Run N_SAMPLES steps of TIME seconds each at CONC concurrency.\n" + "May be used to measure overhead in merlin.\n" + "Iterates thru a chain of workflows.", + ], + [ + "null_spec", + "run N_SAMPLES null steps at CONC concurrency for TIME seconds each. May be used to measure overhead in merlin.", + ], + [ + "openfoam_wf", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" + "using docker.", + ], + [ + "openfoam_wf_no_docker", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and vizualizing OpenFOAM runs\n" + "without using docker.", + ], + [ + "openfoam_wf_singularity", + "A parameter study that includes initializing, running,\n" + "post-processing, collecting, learning and visualizing OpenFOAM runs\n" + "using singularity.", + ], + [ + "optimization_basic", + "Design Optimization Template\n" + "To use,\n" + "1. Specify the first three variables here (N_DIMS, TEST_FUNCTION, DEBUG)\n" + "2. Run the template_config file in current directory using `python template_config.py`\n" + "3. Merlin run as usual (merlin run optimization.yaml)\n" + "* MAX_ITER and the N_SAMPLES options use default values unless using DEBUG mode\n" + "* BOUNDS_X and UNCERTS_X are configured using the template_config.py scripts", + ], + ["remote_feature_demo", "Run 10 hello worlds."], + ["restart", "A simple ensemble of with restarts."], + ["restart_delay", "A simple ensemble of with restart delay times."], + ["simple_chain", "test to see that chains are not run in parallel"], + ["slurm_par", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ["slurm_par_restart", "A simple ensemble of parallel MPI jobs run by slurm (srun)."], + ] + expected = "\n" + tabulate(expected_rows, expected_headers) + "\n" + actual = list_examples() + print(f"actual:\n{actual}") + print(f"expected:\n{expected}") + assert actual == expected + + +def test_setup_example_invalid_name(): + """ + Test the `setup_example` function with an invalid example name. + This should just return None. + """ + assert setup_example("invalid_example_name", None) is None + + +def test_setup_example_no_outdir(examples_testing_dir: str): + """ + Test the `setup_example` function with an invalid example name. + This should create a directory with the example name (in this case hello) + and copy all of the example contents to this folder. + We'll create a directory specifically for this test and move into it so that + the `setup_example` function creates the hello/ subdirectory in a directory with + the name of this test (setup_no_outdir). + + :param examples_testing_dir: The path to the the temp output directory for examples tests + """ + cwd = os.getcwd() + + # Create the temp path to store this setup and move into that directory + setup_example_dir = os.path.join(examples_testing_dir, "setup_no_outdir") + create_dir(setup_example_dir) + os.chdir(setup_example_dir) + + # This should still work and return to us the name of the example + try: + assert setup_example("hello", None) == "hello" + except AssertionError as exc: + os.chdir(cwd) + raise AssertionError from exc + + # All files from this example should be written to a directory with the example name + full_output_path = os.path.join(setup_example_dir, "hello") + expected_files = [ + os.path.join(full_output_path, "hello_samples.yaml"), + os.path.join(full_output_path, "hello.yaml"), + os.path.join(full_output_path, "my_hello.yaml"), + os.path.join(full_output_path, "requirements.txt"), + os.path.join(full_output_path, "make_samples.py"), + ] + try: + for file in expected_files: + assert os.path.exists(file) + except AssertionError as exc: + os.chdir(cwd) + raise AssertionError from exc + finally: + os.chdir(cwd) + + +def test_setup_example_outdir_exists(examples_testing_dir: str): + """ + Test the `setup_example` function with an output directory that already exists. + This should just return None. + + :param examples_testing_dir: The path to the the temp output directory for examples tests + """ + assert setup_example("hello", examples_testing_dir) is None + + +@pytest.mark.parametrize( + "example_name, example_files, expected_return", + [ + ( + "feature_demo", + [ + ".gitignore", + "feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ], + "feature_demo", + ), + ( + "flux_local", + [ + "flux_local.yaml", + "flux_par_restart.yaml", + "flux_par.yaml", + "paper.yaml", + "requirements.txt", + "scripts/flux_info.py", + "scripts/hello_sleep.c", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/paper_workers.sbatch", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + "scripts/workers.bsub", + ], + "flux", + ), + ( + "lsf_par", + [ + "lsf_par_srun.yaml", + "lsf_par.yaml", + "scripts/hello.c", + "scripts/make_samples.py", + ], + "lsf", + ), + ( + "slurm_par", + [ + "slurm_par.yaml", + "slurm_par_restart.yaml", + "requirements.txt", + "scripts/hello.c", + "scripts/make_samples.py", + "scripts/test_workers.sbatch", + "scripts/workers.sbatch", + ], + "slurm", + ), + ( + "hello", + [ + "hello_samples.yaml", + "hello.yaml", + "my_hello.yaml", + "requirements.txt", + "make_samples.py", + ], + "hello", + ), + ( + "hpc_demo", + [ + "hpc_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ], + "hpc_demo", + ), + ( + "iterative_demo", + [ + "iterative_demo.yaml", + "cumulative_sample_processor.py", + "faker_sample.py", + "sample_collector.py", + "sample_processor.py", + "requirements.txt", + ], + "iterative_demo", + ), + ( + "null_spec", + [ + "null_spec.yaml", + "null_chain.yaml", + ".gitignore", + "Makefile", + "requirements.txt", + "scripts/aggregate_chain_output.sh", + "scripts/aggregate_output.sh", + "scripts/check_completion.sh", + "scripts/kill_all.sh", + "scripts/launch_chain_job.py", + "scripts/launch_jobs.py", + "scripts/make_samples.py", + "scripts/read_output_chain.py", + "scripts/read_output.py", + "scripts/search.sh", + "scripts/submit_chain.sbatch", + "scripts/submit.sbatch", + ], + "null_spec", + ), + ( + "openfoam_wf", + [ + "openfoam_wf.yaml", + "openfoam_wf_docker_template.yaml", + "README.md", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf", + ), + ( + "openfoam_wf_no_docker", + [ + "openfoam_wf_no_docker.yaml", + "openfoam_wf_no_docker_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf_no_docker", + ), + ( + "openfoam_wf_singularity", + [ + "openfoam_wf_singularity.yaml", + "openfoam_wf_singularity_template.yaml", + "requirements.txt", + "scripts/make_samples.py", + "scripts/blockMesh_template.txt", + "scripts/cavity_setup.sh", + "scripts/combine_outputs.py", + "scripts/learn.py", + "scripts/mesh_param_script.py", + "scripts/run_openfoam", + ], + "openfoam_wf_singularity", + ), + ( + "optimization_basic", + [ + "optimization_basic.yaml", + "requirements.txt", + "template_config.py", + "template_optimization.temp", + "scripts/collector.py", + "scripts/optimizer.py", + "scripts/test_functions.py", + "scripts/visualizer.py", + ], + "optimization", + ), + ( + "remote_feature_demo", + [ + ".gitignore", + "remote_feature_demo.yaml", + "requirements.txt", + "scripts/features.json", + "scripts/hello_world.py", + "scripts/pgen.py", + ], + "remote_feature_demo", + ), + ("restart", ["restart.yaml", "scripts/make_samples.py"], "restart"), + ("restart_delay", ["restart_delay.yaml", "scripts/make_samples.py"], "restart_delay"), + ], +) +def test_setup_example(examples_testing_dir: str, example_name: str, example_files: List[str], expected_return: str): + """ + Run tests for the `setup_example` function. + Each test will consist of: + 1. The name of the example to setup + 2. A list of files that we're expecting to be setup + 3. The expected return value + Each test is a tuple in the parametrize decorator above this test function. + + :param examples_testing_dir: The path to the the temp output directory for examples tests + :param example_name: The name of the example to setup + :param example_files: A list of filenames that should be copied by setup_example + :param expected_return: The expected return value from `setup_example` + """ + # Create the temp path to store this setup + setup_example_dir = os.path.join(examples_testing_dir, f"setup_{example_name}") + + # Ensure that the example name is returned + actual = setup_example(example_name, setup_example_dir) + assert actual == expected_return + + # Ensure all of the files that should've been copied were copied + expected_files = [os.path.join(setup_example_dir, expected_file) for expected_file in example_files] + for file in expected_files: + assert os.path.exists(file) + + +def test_setup_example_simple_chain(examples_testing_dir: str): + """ + Test the `setup_example` function for the simple_chain example. + This example just writes a single file so we can't run it in the `test_setup_example` test. + + :param examples_testing_dir: The path to the the temp output directory for examples tests + """ + + # Create the temp path to store this setup + output_file = os.path.join(examples_testing_dir, "simple_chain.yaml") + + # Ensure that the example name is returned + actual = setup_example("simple_chain", output_file) + assert actual == "simple_chain" + assert os.path.exists(output_file) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..0b408db54 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,44 @@ +""" +Utility functions for our test suite. +""" + +import os +from typing import Dict + +from tests.constants import SERVER_PASS + + +def create_pass_file(pass_filepath: str): + """ + Check if a password file already exists (it will if the redis server has been started) + and if it hasn't then create one and write the password to the file. + + :param pass_filepath: The path to the password file that we need to check for/create + """ + if not os.path.exists(pass_filepath): + with open(pass_filepath, "w") as pass_file: + pass_file.write(SERVER_PASS) + + +def create_cert_files(cert_filepath: str, cert_files: Dict[str, str]): + """ + Check if cert files already exist and if they don't then create them. + + :param cert_filepath: The path to the cert files + :param cert_files: A dict of certification files to create + """ + for cert_file in cert_files.values(): + full_cert_filepath = f"{cert_filepath}/{cert_file}" + if not os.path.exists(full_cert_filepath): + with open(full_cert_filepath, "w"): + pass + + +def create_dir(dirpath: str): + """ + Check if `dirpath` exists and if it doesn't then create it. + + :param dirpath: The directory to create + """ + if not os.path.exists(dirpath): + os.mkdir(dirpath)