LLNL · bgunnar5 · Oct 28, 2024 · May 4, 2023 · May 4, 2023 · May 4, 2023
diff --git a/.gitignore b/.gitignore
@@ -39,8 +39,9 @@ flux.out
 slurm*.out
 docs/build/
 
-# Tox files
+# Test files
 .tox/*
+.coverage
 
 # Jupyter
 jupyter/.ipynb_checkpoints

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.12.2b1]
+## [1.12.2]
 ### Added
 - Conflict handler option to the `dict_deep_merge` function in `utils.py`
 - Ability to add module-specific pytest fixtures
@@ -19,6 +19,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - New step return code `$(MERLIN_RAISE_ERROR)` to force an error to be raised by a task (mainly for testing)
   - Added description of this to docs
 - New test to ensure a single failed task won't break a workflow
+- Several new unit tests for the following subdirectories:
+  - `merlin/common/`
+  - `merlin/config/`
+  - `merlin/examples/`
+  - `merlin/server/`
+- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures
+  - `RedisServerManager`: context to help with starting/stopping a redis server for tests
+  - `CeleryWorkersManager`: context to help with starting/stopping workers for tests
+- Ability to copy and print the `Config` object from `merlin/config/__init__.py`
+- Equality method to the `ContainerFormatConfig` and `ContainerConfig` objects from `merlin/server/server_util.py`
 
 ### Changed
 - `merlin info` is cleaner and gives python package info
@@ -28,6 +38,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added fixtures for `merlin status` tests that copy the workspace to a temporary directory so you can see exactly what's run in a test
 - Batch block and workers now allow for variables to be used in node settings
 - Task id is now the path to the directory
+- Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier
+- Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier
+- Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot`
 
 ### Fixed
 - Bugfix for output of `merlin example openfoam_wf_singularity`
@@ -95,8 +108,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - this required adding a decent amount of test files to help with the tests; these can be found under the tests/unit/study/status_test_files directory
 - Pytest fixtures in the `conftest.py` file of the integration test suite
   - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command
-- Tests for the `celeryadapter.py` module
-- New CeleryTestWorkersManager context to help with starting/stopping workers for tests
+- Coverage to the test suite. This includes adding tests for:
+  - `merlin/common/`
+  - `merlin/config/`
+  - `merlin/examples/`
+  - `celeryadapter.py`
+- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures
+  - `RedisServerManager`: context to help with starting/stopping a redis server for tests
+  - `CeleryWorkersManager`: context to help with starting/stopping workers for tests
+- Ability to copy and print the `Config` object from `merlin/config/__init__.py`
 
 ### Changed
 - Reformatted the entire `merlin status` command
@@ -132,7 +152,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The `merlin monitor` command will now keep an allocation up if the queues are empty and workers are still processing tasks
 - Add the restart keyword to the specification docs
 - Cyclical imports and config imports that could easily cause ci issues
-
 ## [1.11.1]
 ### Fixed
 - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`)

diff --git a/Makefile b/Makefile
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/config.mk b/config.mk
@@ -18,7 +18,7 @@ else
 endif
 
 VER?=1.0.0
-VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+
+VSTRING=[0-9]\+\.[0-9]\+\.[0-9]\+\(b[0-9]\+\)\?
 YEAR=20[0-9][0-9]
 NEW_YEAR?=2023
 CHANGELOG_VSTRING="## \[$(VSTRING)\]"

diff --git a/merlin/__init__.py b/merlin/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -38,7 +38,7 @@
 import sys
 
 
-__version__ = "1.12.2b1"
+__version__ = "1.12.2"
 VERSION = __version__
 PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "")
 

diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/celery.py b/merlin/celery.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/dumper.py b/merlin/common/dumper.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1
+# This file is part of Merlin, Version: 1.12.2
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py
@@ -8,7 +8,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py
@@ -8,7 +8,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -225,8 +225,8 @@ def __setitem__(self, full_address, sub_tree):
 
         # Replace if we already have something at this address.
         if delete_me is not None:
-            self.children.__delitem__(full_address)
             SampleIndex.check_valid_addresses_for_insertion(full_address, sub_tree)
+            self.children.__delitem__(full_address)
             self.children[full_address] = sub_tree
             return
         raise KeyError

diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -52,11 +52,10 @@ def _get_key_path():
     except AttributeError:
         key_filepath = "~/.merlin/encrypt_data_key"
 
-    try:
-        key_filepath = os.path.abspath(os.path.expanduser(key_filepath))
-    except KeyError as e:
-        raise ValueError("Error! No password provided for RabbitMQ") from e
-    return key_filepath
+    if key_filepath is None:
+        raise ValueError("Error! No password provided for RabbitMQ")
+
+    return os.path.abspath(os.path.expanduser(key_filepath))
 
 
 def _gen_key(key_path):

diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -35,6 +35,7 @@
 import numpy as np
 
 
+# TODO should we move this to merlin-spellbook?
 def scale_samples(samples_norm, limits, limits_norm=(0, 1), do_log=False):
     """Scale samples to new limits, either log10 or linearly.
 

diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -31,7 +31,7 @@
 """
 Used to store the application configuration.
 """
-
+from copy import copy
 from types import SimpleNamespace
 from typing import Dict, List, Optional
 
@@ -56,6 +56,35 @@ def __init__(self, app_dict):
         self.results_backend: Optional[SimpleNamespace]
         self.load_app_into_namespaces(app_dict)
 
+    def __copy__(self):
+        """
+        A magic method to allow this class to be copied with copy(instance_of_Config).
+        """
+        cls = self.__class__
+        result = cls.__new__(cls)
+        copied_attrs = {
+            "celery": copy(self.__dict__["celery"]),
+            "broker": copy(self.__dict__["broker"]),
+            "results_backend": copy(self.__dict__["results_backend"]),
+        }
+        result.__dict__.update(copied_attrs)
+        return result
+
+    def __str__(self):
+        """
+        A magic method so we can print the CONFIG class.
+        """
+        formatted_str = "config:"
+        attrs = {"celery": self.celery, "broker": self.broker, "results_backend": self.results_backend}
+        for name, attr in attrs.items():
+            if attr is not None:
+                items = (f"    {k}: {v!r}" for k, v in attr.__dict__.items())
+                joined_items = "\n".join(items)
+                formatted_str += f"\n  {name}:\n{joined_items}"
+            else:
+                formatted_str += f"\n  {name}:\n    None"
+        return formatted_str
+
     def load_app_into_namespaces(self, app_dict: Dict) -> None:
         """
         Makes the application dictionary into a namespace, sets the attributes of the Config from the namespace values.

diff --git a/merlin/config/broker.py b/merlin/config/broker.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -85,13 +85,13 @@ def get_rabbit_connection(include_password, conn="amqps"):
         password_filepath = CONFIG.broker.password
         LOG.debug(f"Broker: password filepath = {password_filepath}")
         password_filepath = os.path.abspath(expanduser(password_filepath))
-    except KeyError as e:  # pylint: disable=C0103
-        raise ValueError("Broker: No password provided for RabbitMQ") from e
+    except (AttributeError, KeyError) as exc:
+        raise ValueError("Broker: No password provided for RabbitMQ") from exc
 
     try:
         password = read_file(password_filepath)
-    except IOError as e:  # pylint: disable=C0103
-        raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from e
+    except IOError as exc:
+        raise ValueError(f"Broker: RabbitMQ password file {password_filepath} does not exist") from exc
 
     try:
         port = CONFIG.broker.port
@@ -205,12 +205,6 @@ def get_connection_string(include_password=True):
     except AttributeError:
         broker = ""
 
-    try:
-        config_path = CONFIG.celery.certs
-        config_path = os.path.abspath(os.path.expanduser(config_path))
-    except AttributeError:
-        config_path = None
-
     if broker not in BROKERS:
         raise ValueError(f"Error: {broker} is not a supported broker.")
     return _sort_valid_broker(broker, include_password)

diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py
@@ -10,7 +10,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #

diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #