From 0f8e937a8b3f38d9516d4f9464457b10013a6dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Juli=C3=A1n=20Espina?= Date: Tue, 25 Jun 2024 13:23:35 -0600 Subject: [PATCH 1/6] chore: initialize `slurm-ops` library --- charmcraft.yaml | 83 ++------------------- dev-requirements.txt | 4 + lib/charms/hpc_libs/v0/slurm_ops.py | 32 ++++++++ pyproject.toml | 8 +- src/charm.py | 110 +++++----------------------- tox.ini | 18 +++-- 6 files changed, 76 insertions(+), 179 deletions(-) create mode 100644 dev-requirements.txt create mode 100644 lib/charms/hpc_libs/v0/slurm_ops.py diff --git a/charmcraft.yaml b/charmcraft.yaml index fca04fa..ba70f4b 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -1,39 +1,14 @@ -# This file configures Charmcraft. -# See https://juju.is/docs/sdk/charmcraft-config for guidance. +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. -# (Required) -# The charm package name, no spaces -# See https://juju.is/docs/sdk/naming#heading--naming-charms for guidance. name: hpc-libs - - -# (Required) -# The charm type, either 'charm' or 'bundle'. -type: charm - - -# (Recommended) -title: Charm Template - - -# (Required) -summary: A very short one-line summary of the charm. - - -# (Required) +title: HPC Libs +summary: Collection of Charm libraries to manage HPC related services. description: | - A single sentence that says what the charm is, concisely and memorably. - - A paragraph of one to three short sentences, that describe what the charm does. - - A third paragraph that explains what need the charm meets. - - Finally, a paragraph that describes whom the charm is useful for. - - -# (Required for 'charm' type) -# A list of environments (OS version and architecture) where charms must be -# built on and run on. + A placeholder charm that contains helpful charm libraries curated by the + HPC team for use when authoring charms that need to manage HPC related services; + Slurm, Munge, etc. +type: charm bases: - build-on: - name: ubuntu @@ -41,45 +16,3 @@ bases: run-on: - name: ubuntu channel: "22.04" - - -# (Optional) Configuration options for the charm -# This config section defines charm config options, and populates the Configure -# tab on Charmhub. -# More information on this section at https://juju.is/docs/sdk/charmcraft-yaml#heading--config -# General configuration documentation: https://juju.is/docs/sdk/config -config: - options: - # An example config option to customise the log level of the workload - log-level: - description: | - Configures the log level of gunicorn. - - Acceptable values are: "info", "debug", "warning", "error" and "critical" - default: "info" - type: string - - -# The containers and resources metadata apply to Kubernetes charms only. -# See https://juju.is/docs/sdk/metadata-reference for a checklist and guidance. -# Remove them if not required. - - -# Your workload’s containers. -containers: - httpbin: - resource: httpbin-image - - -# This field populates the Resources tab on Charmhub. -resources: - # An OCI image resource for each container listed above. - # You may remove this if your charm will run without a workload sidecar container. - httpbin-image: - type: oci-image - description: OCI image for httpbin - # The upstream-source field is ignored by Juju. It is included here as a - # reference so the integration testing suite knows which image to deploy - # during testing. This field is also used by the 'canonical/charming-actions' - # Github action for automated releasing. - upstream-source: kennethreitz/httpbin diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..4798c8e --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,4 @@ +juju ~= 3.3 +pytest ~= 7.2 +pytest-operator ~= 0.34 +pytest-order ~= 1.1 \ No newline at end of file diff --git a/lib/charms/hpc_libs/v0/slurm_ops.py b/lib/charms/hpc_libs/v0/slurm_ops.py new file mode 100644 index 0000000..0c81941 --- /dev/null +++ b/lib/charms/hpc_libs/v0/slurm_ops.py @@ -0,0 +1,32 @@ +"""TODO: Add a proper docstring here. + +This is a placeholder docstring for this charm library. Docstrings are +presented on Charmhub and updated whenever you push a new version of the +library. + +Complete documentation about creating and documenting libraries can be found +in the SDK docs at https://juju.is/docs/sdk/libraries. + +See `charmcraft publish-lib` and `charmcraft fetch-lib` for details of how to +share and consume charm libraries. They serve to enhance collaboration +between charmers. Use a charmer's libraries for classes that handle +integration with their charm. + +Bear in mind that new revisions of the different major API versions (v0, v1, +v2 etc) are maintained independently. You can continue to update v0 and v1 +after you have pushed v3. + +Markdown is supported, following the CommonMark specification. +""" + +# The unique Charmhub library identifier, never change it +LIBID = "541fd767f90b40539cf7cd6e7db8fabf" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 1 + +# TODO: add your code here! Happy coding! diff --git a/pyproject.toml b/pyproject.toml index e10531c..b751b3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,9 @@ target-version = ["py38"] # Linting tools configuration [tool.ruff] line-length = 99 +extend-exclude = ["__pycache__", "*.egg_info"] + +[tool.ruff.lint] select = ["E", "W", "F", "C", "N", "D", "I001"] extend-ignore = [ "D203", @@ -32,11 +35,8 @@ extend-ignore = [ "D413", ] ignore = ["E501", "D107"] -extend-exclude = ["__pycache__", "*.egg_info"] per-file-ignores = {"tests/*" = ["D100","D101","D102","D103","D104"]} - -[tool.ruff.mccabe] -max-complexity = 10 +mccabe = { "max-complexity" = 10} [tool.codespell] skip = "build,lib,venv,icon.svg,.tox,.git,.mypy_cache,.ruff_cache,.coverage" diff --git a/src/charm.py b/src/charm.py index d9aee3e..3e3da86 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1,104 +1,28 @@ #!/usr/bin/env python3 -# Copyright 2024 Jason C. Nucciarone +# Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. -# -# Learn more at: https://juju.is/docs/sdk -"""Charm the service. -Refer to the following tutorial that will help you -develop a new k8s charm using the Operator Framework: +"""A placeholder charm for the HPC Libs.""" -https://juju.is/docs/sdk/create-a-minimal-kubernetes-charm -""" +from ops import BlockedStatus +from ops.charm import CharmBase, StartEvent +from ops.main import main -import logging -from typing import cast -import ops +class HPCLibsCharm(CharmBase): + """Placeholder charm for HPC Libs.""" -# Log messages can be retrieved using juju debug-log -logger = logging.getLogger(__name__) + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.framework.observe(self.on.start, self._on_start) -VALID_LOG_LEVELS = ["info", "debug", "warning", "error", "critical"] + def _on_start(self, _: StartEvent) -> None: + """Handle start event.""" + self.unit.status = BlockedStatus( + "hpc-libs is not meant to be deployed as a standalone charm" + ) -class HpcLibsCharm(ops.CharmBase): - """Charm the service.""" - - def __init__(self, framework: ops.Framework): - super().__init__(framework) - framework.observe(self.on["httpbin"].pebble_ready, self._on_httpbin_pebble_ready) - framework.observe(self.on.config_changed, self._on_config_changed) - - def _on_httpbin_pebble_ready(self, event: ops.PebbleReadyEvent): - """Define and start a workload using the Pebble API. - - Change this example to suit your needs. You'll need to specify the right entrypoint and - environment configuration for your specific workload. - - Learn more about interacting with Pebble at at https://juju.is/docs/sdk/pebble. - """ - # Get a reference the container attribute on the PebbleReadyEvent - container = event.workload - # Add initial Pebble config layer using the Pebble API - container.add_layer("httpbin", self._pebble_layer, combine=True) - # Make Pebble reevaluate its plan, ensuring any services are started if enabled. - container.replan() - # Learn more about statuses in the SDK docs: - # https://juju.is/docs/sdk/constructs#heading--statuses - self.unit.status = ops.ActiveStatus() - - def _on_config_changed(self, event: ops.ConfigChangedEvent): - """Handle changed configuration. - - Change this example to suit your needs. If you don't need to handle config, you can remove - this method. - - Learn more about config at https://juju.is/docs/sdk/config - """ - # Fetch the new config value - log_level = cast(str, self.model.config["log-level"]).lower() - - # Do some validation of the configuration option - if log_level in VALID_LOG_LEVELS: - # The config is good, so update the configuration of the workload - container = self.unit.get_container("httpbin") - # Verify that we can connect to the Pebble API in the workload container - if container.can_connect(): - # Push an updated layer with the new config - container.add_layer("httpbin", self._pebble_layer, combine=True) - container.replan() - - logger.debug("Log level for gunicorn changed to '%s'", log_level) - self.unit.status = ops.ActiveStatus() - else: - # We were unable to connect to the Pebble API, so we defer this event - event.defer() - self.unit.status = ops.WaitingStatus("waiting for Pebble API") - else: - # In this case, the config option is bad, so block the charm and notify the operator. - self.unit.status = ops.BlockedStatus("invalid log level: '{log_level}'") - - @property - def _pebble_layer(self) -> ops.pebble.LayerDict: - """Return a dictionary representing a Pebble layer.""" - return { - "summary": "httpbin layer", - "description": "pebble config layer for httpbin", - "services": { - "httpbin": { - "override": "replace", - "summary": "httpbin", - "command": "gunicorn -b 0.0.0.0:80 httpbin:app -k gevent", - "startup": "enabled", - "environment": { - "GUNICORN_CMD_ARGS": f"--log-level {self.model.config['log-level']}" - }, - } - }, - } - - -if __name__ == "__main__": # pragma: nocover - ops.main(HpcLibsCharm) # type: ignore +if __name__ == "__main__": + main(HPCLibsCharm) diff --git a/tox.ini b/tox.ini index 6485e9d..4676901 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,4 @@ -# Copyright 2024 Jason C. Nucciarone +# Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. [tox] @@ -10,7 +10,7 @@ min_version = 4.0.0 [vars] src_path = {tox_root}/src tests_path = {tox_root}/tests -;lib_path = {tox_root}/lib/charms/operator_name_with_underscores +lib_path = {tox_root}/lib/charms/hpc_libs all_path = {[vars]src_path} {[vars]tests_path} [testenv] @@ -30,7 +30,7 @@ deps = ruff commands = black {[vars]all_path} - ruff --fix {[vars]all_path} + ruff check --fix {[vars]all_path} [testenv:lint] description = Check code against coding style standards @@ -43,7 +43,7 @@ commands = # and uncomment the following line # codespell {[vars]lib_path} codespell {tox_root} - ruff {[vars]all_path} + ruff check {[vars]all_path} black --check --diff {[vars]all_path} [testenv:unit] @@ -73,9 +73,7 @@ commands = [testenv:integration] description = Run integration tests deps = - pytest - juju - pytest-operator + -r {tox_root}/dev-requirements.txt -r {tox_root}/requirements.txt commands = pytest -v \ @@ -84,3 +82,9 @@ commands = --log-cli-level=INFO \ {posargs} \ {[vars]tests_path}/integration + +[testenv:venv] +deps = + -r {tox_root}/dev-requirements.txt + -r {tox_root}/requirements.txt +commands = {posargs} \ No newline at end of file From 3afb2ccec1281ee1c8a97b9f9b6dd7b269c0eb82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Juli=C3=A1n=20Espina?= Date: Tue, 25 Jun 2024 17:49:54 -0600 Subject: [PATCH 2/6] feat: Implement slurm ops library Some complex tempfile manipulations were implemented to be able to get a new key without overriding the current key, but it's not too bad. --- lib/charms/hpc_libs/v0/slurm_ops.py | 207 ++++++++++++++++++++++++++-- 1 file changed, 192 insertions(+), 15 deletions(-) diff --git a/lib/charms/hpc_libs/v0/slurm_ops.py b/lib/charms/hpc_libs/v0/slurm_ops.py index 0c81941..5fa0f8a 100644 --- a/lib/charms/hpc_libs/v0/slurm_ops.py +++ b/lib/charms/hpc_libs/v0/slurm_ops.py @@ -1,24 +1,67 @@ -"""TODO: Add a proper docstring here. +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -This is a placeholder docstring for this charm library. Docstrings are -presented on Charmhub and updated whenever you push a new version of the -library. -Complete documentation about creating and documenting libraries can be found -in the SDK docs at https://juju.is/docs/sdk/libraries. +"""Library to manage the Slurm snap. -See `charmcraft publish-lib` and `charmcraft fetch-lib` for details of how to -share and consume charm libraries. They serve to enhance collaboration -between charmers. Use a charmer's libraries for classes that handle -integration with their charm. +This library contains the `SlurmManager` class, which offers interfaces to use and manage +the Slurm snap inside charms. -Bear in mind that new revisions of the different major API versions (v0, v1, -v2 etc) are maintained independently. You can continue to update v0 and v1 -after you have pushed v3. +### General usage -Markdown is supported, following the CommonMark specification. +For starters, the `SlurmManager` constructor receives a `Service` enum as a parameter, which +helps the manager determine things like the correct service to enable, or the correct settings +key to mutate. + +``` +from charms.hpc_libs.v0.slurm_ops import ( + Service, + SlurmManager, +) + +class ApplicationCharm(CharmBase): + # Application charm that needs to use the Slurm snap. + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + # Charm events defined in the NFSRequires class. + self._slurm_manager = SlurmManager(Service.SLURMCTLD) + self.framework.observe( + self.on.install, + self._on_install, + ) + + def _on_install(self, _) -> None: + self._slurm_manager.install() + self.unit.set_workload_version(self._slurm_manager.version()) + self._slurm_manager.set_config("cluster-name", "cluster") +``` """ +import base64 +import enum +import functools +import logging +import os +import subprocess +import tempfile + +import yaml + +_logger = logging.getLogger(__name__) + # The unique Charmhub library identifier, never change it LIBID = "541fd767f90b40539cf7cd6e7db8fabf" @@ -29,4 +72,138 @@ # to 0 if you are raising the major API version LIBPATCH = 1 -# TODO: add your code here! Happy coding! + +PYDEPS = ["pyyaml>=6.0.1"] + + +def _call(cmd: str, *args: [str]) -> bytes: + """Call a command with logging. + + Raises: + subprocess.CalledProcessError: Raised if the command fails. + """ + cmd = [cmd, *args] + _logger.debug(f"Executing command {cmd}") + try: + return subprocess.check_output(cmd, stderr=subprocess.PIPE, text=False) + except subprocess.CalledProcessError as e: + _logger.error(f"`{' '.join(cmd)}` failed") + _logger.error(f"stderr: {e.stderr.decode()}") + raise + + +def _snap(*args) -> str: + """Control snap by via executed `snap ...` commands. + + Raises: + subprocess.CalledProcessError: Raised if snap command fails. + """ + return _call("snap", *args).decode() + + +_get_config = functools.partial(_snap, "get", "slurm") +_set_config = functools.partial(_snap, "set", "slurm") + + +class Service(enum.Enum): + """Type of Slurm service that will be managed by `SlurmManager`.""" + + SLURMD = "slurmd" + SLURMCTLD = "slurmctld" + SLURMDBD = "slurmdbd" + SLURMRESTD = "slurmrestd" + + @property + def config_name(self) -> str: + """Configuration name on the slurm snap for this service type.""" + if self is Service.SLURMCTLD: + return "slurm" + return self.value + + +class SlurmManager: + """Slurm snap manager. + + This class offers methods to manage the Slurm snap for a certain service type. + The list of available services is specified by the `Service` enum. + """ + + def __init__(self, service: Service): + self._service = service + + def install(self): + """Install the slurm snap in this system.""" + # TODO: Pin slurm to the stable channel + _snap("install", "slurm", "--channel", "latest/candidate", "--classic") + + def start(self): + """Start and enables the managed slurm service and the munged service.""" + _snap("start", "--enable", "slurm.munged") + _snap("start", "--enable", f"slurm.{self._service.value}") + + def restart(self): + """Restart the managed slurm service.""" + _snap("restart", f"slurm.{self._service.value}") + + def restart_munged(self): + """Restart the munged service.""" + _snap("restart", "slurm.munged") + + def disable(self): + """Disable the managed slurm service and the munged service.""" + _snap("stop", "--disable", "slurm.munged") + _snap("stop", "--disable", f"slurm.{self._service.value}") + + def set_config(self, key: str, value: str): + """Set a snap config for the managed slurm service. + + See the configuration section from the [Slurm readme](https://github.com/charmed-hpc/slurm-snap#configuration) + for a list of all the available configurations. + + Note that this will only allow configuring the settings that are exclusive to + the specific managed service. (the slurmctld service uses the slurm parent key) + """ + _set_config(f"{self._service.config_name}.{key}={value}") + + def get_config(self, key: str) -> str: + """Get a snap config for the managed slurm service. + + See the configuration section from the [Slurm readme](https://github.com/charmed-hpc/slurm-snap#configuration) + for a list of all the available configurations. + + Note that this will only allow fetching the settings that are exclusive to + the specific managed service. (the slurmctld service uses the slurm parent key) + """ + # Snap returns the config value with an additional newline at the end. + return _get_config(f"{self._service.config_name}.{key}").strip() + + def generate_munge_key(self) -> bytes: + """Generate a new cryptographically secure munged key.""" + handle, path = tempfile.mkstemp() + try: + _call("mungekey", "-f", "-k", path) + os.close(handle) + with open(path, "rb") as f: + return f.read() + finally: + os.remove(path) + + def set_munge_key(self, key: bytes): + """Set the current munged key.""" + # TODO: use `slurm.setmungekey` when implemented + # subprocess.run(["slurm.setmungekey"], stdin=key) + key = base64.b64encode(key).decode() + _set_config(f"munge.key={key}") + + def get_munge_key(self) -> bytes: + """Get the current munged key.""" + # TODO: use `slurm.setmungekey` when implemented + # key = subprocess.run(["slurm.getmungekey"]) + key = _get_config("munge.key") + return base64.b64decode(key) + + def version(self) -> str: + """Get the installed Slurm version of the snap.""" + info = yaml.safe_load(_snap("info", "slurm")) + version: str = info["installed"] + return version.split(maxsplit=1)[0] From 6afa24656371080007d5e2ff3aae3d02c24cfc90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Juli=C3=A1n=20Espina?= Date: Fri, 28 Jun 2024 12:59:25 -0600 Subject: [PATCH 3/6] tests: add integration tests for `slurm_ops` --- .github/workflows/ci.yaml | 52 ++++++++++++++++ dev-requirements.txt | 2 - tests/integration/slurm_ops/test_manager.py | 68 +++++++++++++++++++++ tests/integration/test_charm.py | 35 ----------- tests/integration/test_hpc_libs.yaml | 43 +++++++++++++ tests/unit/test_charm.py | 68 --------------------- tox.ini | 15 ++--- 7 files changed, 167 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 tests/integration/slurm_ops/test_manager.py delete mode 100644 tests/integration/test_charm.py create mode 100644 tests/integration/test_hpc_libs.yaml delete mode 100644 tests/unit/test_charm.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..70d76f4 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,52 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +name: cephfs-server-proxy tests +on: + workflow_call: + pull_request: + +jobs: + inclusive-naming-check: + name: Inclusive naming check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Run tests + uses: get-woke/woke-action@v0 + with: + fail-on-error: true + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install dependencies + run: python3 -m pip install tox + - name: Run linters + run: tox -e lint + + integration-test: + name: Integration tests + runs-on: ubuntu-latest + needs: + - inclusive-naming-check + - lint + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up LXD + uses: canonical/setup-lxd@v0.1.1 + with: + channel: 5.21/stable + - name: Set up gambol + run: | + wget https://github.com/NucciTheBoss/gambol/releases/download/v0.1.0-rc2/gambol_0.1.0_amd64-rc2.snap + sudo snap install ./gambol_*.snap --dangerous + sudo snap connect gambol:lxd lxd:lxd + sudo snap connect gambol:dot-gambol + - name: Run tests + run: tox -e integration diff --git a/dev-requirements.txt b/dev-requirements.txt index 4798c8e..5960b53 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,2 @@ -juju ~= 3.3 pytest ~= 7.2 -pytest-operator ~= 0.34 pytest-order ~= 1.1 \ No newline at end of file diff --git a/tests/integration/slurm_ops/test_manager.py b/tests/integration/slurm_ops/test_manager.py new file mode 100644 index 0000000..e308c3b --- /dev/null +++ b/tests/integration/slurm_ops/test_manager.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + + +import pytest + +from lib.charms.hpc_libs.v0.slurm_ops import Service, SlurmManager + + +@pytest.fixture +def slurm_manager() -> SlurmManager: + return SlurmManager(Service.SLURMCTLD) + + +@pytest.mark.order(1) +def test_install(slurm_manager: SlurmManager) -> None: + """Install Slurm using the manager.""" + slurm_manager.install() + slurm_manager.start() + slurm_manager.set_munge_key(slurm_manager.generate_munge_key()) + + with open("/var/snap/slurm/common/etc/munge/munge.key", "rb") as f: + key: bytes = f.read() + + assert key == slurm_manager.get_munge_key() + + +@pytest.mark.order(2) +def test_rotate_key(slurm_manager: SlurmManager) -> None: + """Test that the munge key can be rotated.""" + old_key = slurm_manager.get_munge_key() + + slurm_manager.set_munge_key(slurm_manager.generate_munge_key()) + + new_key = slurm_manager.get_munge_key() + + assert old_key != new_key + + +@pytest.mark.order(3) +def test_slurm_config(slurm_manager: SlurmManager) -> None: + """Test that the slurm config can be changed.""" + slurm_manager.set_config("cluster-name", "test-cluster") + + value = slurm_manager.get_config("cluster-name") + + assert value == "test-cluster" + + with open("/var/snap/slurm/common/etc/slurm/slurm.conf", "r") as f: + output = f.read() + + for line in output.splitlines(): + entry = line.split("=") + if len(entry) != 2: + continue + key, value = entry + if key == "ClusterName": + assert value == "test-cluster" + + +@pytest.mark.order(4) +def test_version(slurm_manager: SlurmManager) -> None: + """Test that the Slurm manager can report its version.""" + version = slurm_manager.version() + + # We are interested in knowing that this does not return a falsy value (`None`, `''`, `[]`, etc.) + assert version diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py deleted file mode 100644 index fdded71..0000000 --- a/tests/integration/test_charm.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Jason C. Nucciarone -# See LICENSE file for licensing details. - -import asyncio -import logging -from pathlib import Path - -import pytest -import yaml -from pytest_operator.plugin import OpsTest - -logger = logging.getLogger(__name__) - -METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) -APP_NAME = METADATA["name"] - - -@pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest): - """Build the charm-under-test and deploy it together with related charms. - - Assert on the unit status before any relations/configurations take place. - """ - # Build and deploy charm from local source folder - charm = await ops_test.build_charm(".") - resources = {"httpbin-image": METADATA["resources"]["httpbin-image"]["upstream-source"]} - - # Deploy the charm and wait for active/idle status - await asyncio.gather( - ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME), - ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=1000 - ), - ) diff --git a/tests/integration/test_hpc_libs.yaml b/tests/integration/test_hpc_libs.yaml new file mode 100644 index 0000000..7c30274 --- /dev/null +++ b/tests/integration/test_hpc_libs.yaml @@ -0,0 +1,43 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "HPC lib tests" +provider: + lxd: +acts: + test-slurm-ops: + name: "Test the slurm_ops library" + run-on: jammy + input: + - host-path: lib + path: lib + - host-path: dev-requirements.txt + path: dev-requirements.txt + - host-path: tests/integration/slurm_ops + path: slurm_ops + scenes: + - name: "Install dependencies in a virtual environment" + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt update + sudo apt install -y python3-venv python3-yaml + python3 -m venv venv --system-site-packages + venv/bin/python3 -m pip install -r dev-requirements.txt + - name: "Run integration tests with pytest" + run: | + sudo venv/bin/python3 -m pytest -v \ + -s \ + --tb native \ + --log-cli-level=INFO \ + slurm_ops diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py deleted file mode 100644 index 9a4b36c..0000000 --- a/tests/unit/test_charm.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2024 Jason C. Nucciarone -# See LICENSE file for licensing details. -# -# Learn more about testing at: https://juju.is/docs/sdk/testing - -import unittest - -import ops -import ops.testing -from charm import HpcLibsCharm - - -class TestCharm(unittest.TestCase): - def setUp(self): - self.harness = ops.testing.Harness(HpcLibsCharm) - self.addCleanup(self.harness.cleanup) - self.harness.begin() - - def test_httpbin_pebble_ready(self): - # Expected plan after Pebble ready with default config - expected_plan = { - "services": { - "httpbin": { - "override": "replace", - "summary": "httpbin", - "command": "gunicorn -b 0.0.0.0:80 httpbin:app -k gevent", - "startup": "enabled", - "environment": {"GUNICORN_CMD_ARGS": "--log-level info"}, - } - }, - } - # Simulate the container coming up and emission of pebble-ready event - self.harness.container_pebble_ready("httpbin") - # Get the plan now we've run PebbleReady - updated_plan = self.harness.get_container_pebble_plan("httpbin").to_dict() - # Check we've got the plan we expected - self.assertEqual(expected_plan, updated_plan) - # Check the service was started - service = self.harness.model.unit.get_container("httpbin").get_service("httpbin") - self.assertTrue(service.is_running()) - # Ensure we set an ActiveStatus with no message - self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) - - def test_config_changed_valid_can_connect(self): - # Ensure the simulated Pebble API is reachable - self.harness.set_can_connect("httpbin", True) - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "debug"}) - # Get the plan now we've run PebbleReady - updated_plan = self.harness.get_container_pebble_plan("httpbin").to_dict() - updated_env = updated_plan["services"]["httpbin"]["environment"] - # Check the config change was effective - self.assertEqual(updated_env, {"GUNICORN_CMD_ARGS": "--log-level debug"}) - self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) - - def test_config_changed_valid_cannot_connect(self): - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "debug"}) - # Check the charm is in WaitingStatus - self.assertIsInstance(self.harness.model.unit.status, ops.WaitingStatus) - - def test_config_changed_invalid(self): - # Ensure the simulated Pebble API is reachable - self.harness.set_can_connect("httpbin", True) - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "foobar"}) - # Check the charm is in BlockedStatus - self.assertIsInstance(self.harness.model.unit.status, ops.BlockedStatus) diff --git a/tox.ini b/tox.ini index 4676901..879a75e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ min_version = 4.0.0 src_path = {tox_root}/src tests_path = {tox_root}/tests lib_path = {tox_root}/lib/charms/hpc_libs -all_path = {[vars]src_path} {[vars]tests_path} +all_path = {[vars]src_path} {[vars]tests_path} {[vars]lib_path} [testenv] set_env = @@ -72,19 +72,12 @@ commands = [testenv:integration] description = Run integration tests -deps = - -r {tox_root}/dev-requirements.txt - -r {tox_root}/requirements.txt +allowlist_externals = gambol commands = - pytest -v \ - -s \ - --tb native \ - --log-cli-level=INFO \ - {posargs} \ - {[vars]tests_path}/integration + gambol -v run tests/integration/test_hpc_libs.yaml [testenv:venv] deps = -r {tox_root}/dev-requirements.txt -r {tox_root}/requirements.txt -commands = {posargs} \ No newline at end of file +commands = {posargs} From beb8e9aa893c76922774c58d4a3b913612260b31 Mon Sep 17 00:00:00 2001 From: Jason Nucciarone <40342202+NucciTheBoss@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:32:53 -0400 Subject: [PATCH 4/6] fix(ci): change name to `hpc-libs tests` --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 70d76f4..88e196e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. -name: cephfs-server-proxy tests +name: hpc-libs tests on: workflow_call: pull_request: From 9414951b118c9d11a5eeaf83c0b4da0aaa9e3e59 Mon Sep 17 00:00:00 2001 From: Jason Nucciarone <40342202+NucciTheBoss@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:49:42 -0400 Subject: [PATCH 5/6] refactor: remove calls to sudo --- tests/integration/test_hpc_libs.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_hpc_libs.yaml b/tests/integration/test_hpc_libs.yaml index 7c30274..b9b22c0 100644 --- a/tests/integration/test_hpc_libs.yaml +++ b/tests/integration/test_hpc_libs.yaml @@ -30,13 +30,13 @@ acts: - name: "Install dependencies in a virtual environment" run: | export DEBIAN_FRONTEND=noninteractive - sudo apt update - sudo apt install -y python3-venv python3-yaml + apt update + apt install -y python3-venv python3-yaml python3 -m venv venv --system-site-packages venv/bin/python3 -m pip install -r dev-requirements.txt - name: "Run integration tests with pytest" run: | - sudo venv/bin/python3 -m pytest -v \ + venv/bin/python3 -m pytest -v \ -s \ --tb native \ --log-cli-level=INFO \ From a7fec9ef86358a70c136b5d85510366b40d5b413 Mon Sep 17 00:00:00 2001 From: Jason Nucciarone <40342202+NucciTheBoss@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:57:20 -0400 Subject: [PATCH 6/6] fix(ci): install `tox` in integration tests --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 88e196e..dc6ef8a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,6 +38,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Install dependencies + run: python3 -m pip install tox - name: Set up LXD uses: canonical/setup-lxd@v0.1.1 with: