Add Software RAID mode validation test using mdadm (New) (#1123)

* Add: check multiple devices (software RAID)

adding new test cases to validate the mode of multiple devices (software
RAID)

---------

Co-authored-by: Pierre Equoy <[email protected]>

checkbox-core-snap/series20/snap/snapcraft.yaml

@@ -337,6 +337,7 @@ parts:
       - wget
       - wmctrl
       - xz-utils
+      - mdadm
       - on armhf:
         - python3-rpi.gpio # only in focal
       - on arm64:

checkbox-core-snap/series22/snap/snapcraft.yaml

@@ -341,6 +341,7 @@ parts:
       - wmctrl
       - xz-utils
       - rt-tests # For realtime performance test
+      - mdadm
       - on armhf:
         - python3-rpi.gpio # only in focal
       - on arm64:

providers/base/bin/check_software_raid.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# This file is part of Checkbox.
+#
+# Copyright 2024 Canonical Ltd.
+# Written by:
+#   Stanley Huang <[email protected]>
+#
+# Checkbox is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3,
+# as published by the Free Software Foundation.
+#
+# Checkbox is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Checkbox. If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import sys
+import logging
+import argparse
+import shlex
+import subprocess
+from pathlib import Path
+from typing import Dict
+
+
+class RAIDStats(Dict):
+    device = str
+    mode = str
+
+
+def get_md_stat(filename: str = "/proc/mdstat") -> list:
+    """
+    Parse the information of RAID devices from /dev/mdstat
+
+    Returns:
+        raid_stats (list): the RAID information
+    """
+    pattern = r"^(md[0-9]+) : active ([a-z0-9]+) [a-z0-9 \[\]]+$"
+    raid_raw_data = ""
+    raid_stats = []
+
+    with Path(filename) as node:
+        raid_raw_data = node.read_text().strip("\n")
+
+    for tmp_data in raid_raw_data.split("\n"):
+        find_result = re.search(pattern, tmp_data)
+        if find_result:
+            raid_node = RAIDStats(
+                device=find_result.groups()[0],
+                mode=find_result.groups()[1]
+            )
+            raid_stats.append(raid_node)
+
+    return raid_stats
+
+
+def dump_raid_info(nodes: list) -> None:
+    """
+    Display detail information for all of RAID devices
+
+    Args:
+        nodes (list): the name of MD devices
+    """
+    for node in nodes:
+        subprocess.run(shlex.split("mdadm -D /dev/{}".format(node)))
+
+
+def check_raid_mode_test(modes: str) -> None:
+    """
+    Validate the RAID modes running on the system is expected
+
+    Args:
+        modes (str): the expected RAID modes on the system
+
+    Raises:
+        ValueError: when if RAID modes are not expected
+    """
+    expected_modes = modes.strip().split()
+    cur_raid_stats = get_md_stat()
+
+    active_mode = [stat["mode"] for stat in cur_raid_stats]
+    logging.info("Active RAID mode on the system: %s", active_mode)
+    logging.info("Expected RAID mode: %s", expected_modes)
+
+    dump_raid_info(
+        [stat["device"] for stat in cur_raid_stats]
+    )
+
+    if sorted(expected_modes) == sorted(active_mode):
+        logging.info("PASS: RAID mode on the system is expected")
+    else:
+        raise ValueError("RAID mode on the system is not expected")
+
+
+def register_arguments():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Check RAID stats and compare mode is expected')
+    parser.add_argument(
+        "--mode",
+        required=True,
+        type=str,
+        help="The RAID mode is enabeld on the running system",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == "__main__":
+
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.INFO)
+    logger_format = "%(asctime)s %(levelname)-8s %(message)s"
+    date_format = "%Y-%m-%d %H:%M:%S"
+
+    # Log DEBUG and INFO to stdout, others to stderr
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    stdout_handler.setFormatter(logging.Formatter(logger_format, date_format))
+
+    stderr_handler = logging.StreamHandler(sys.stderr)
+    stderr_handler.setFormatter(logging.Formatter(logger_format, date_format))
+
+    stdout_handler.setLevel(logging.DEBUG)
+    stderr_handler.setLevel(logging.WARNING)
+
+    # Add a filter to the stdout handler to limit log records to
+    # INFO level and below
+    stdout_handler.addFilter(lambda record: record.levelno <= logging.INFO)
+
+    root_logger.addHandler(stderr_handler)
+    root_logger.addHandler(stdout_handler)
+
+    args = register_arguments()
+
+    try:
+        check_raid_mode_test(args.mode)
+    except Exception as err:
+        logging.error(err)

providers/base/tests/test_check_software_raid.py

@@ -0,0 +1,91 @@
+import sys
+import unittest
+from unittest.mock import patch
+import check_software_raid
+
+
+class TestCheckSoftwareRAID(unittest.TestCase):
+
+    def test_get_md_stat_intel_raid(self):
+
+        expected_data = [
+            {"device": "md126", "mode": "raid0"}
+        ]
+        raid_data = check_software_raid.get_md_stat(
+            "tests/test_data/mdstat_intel_rst.txt"
+        )
+        self.assertListEqual(raid_data, expected_data)
+
+    def test_get_md_stat_multiple_raid(self):
+
+        expected_data = [
+            {"device": "md2", "mode": "raid1"},
+            {"device": "md1", "mode": "raid0"}
+        ]
+        raid_data = check_software_raid.get_md_stat(
+            "tests/test_data/mdstat_multiple_raid.txt"
+        )
+        self.assertListEqual(raid_data, expected_data)
+
+    def test_get_md_stat_empty(self):
+
+        raid_data = check_software_raid.get_md_stat(
+            "tests/test_data/mdstat_none_raid.txt"
+        )
+        self.assertListEqual(raid_data, [])
+
+    @patch("check_software_raid.dump_raid_info")
+    @patch("check_software_raid.get_md_stat")
+    def test_check_raid_mode_is_expected(self, mock_get_md, mock_dump_raid):
+
+        mock_get_md.return_value = [
+            {"device": "md2", "mode": "raid1"},
+            {"device": "md1", "mode": "raid0"}
+        ]
+
+        check_software_raid.check_raid_mode_test("raid1 raid0")
+        mock_get_md.assert_called_with()
+        mock_dump_raid.assert_called_with(["md2", "md1"])
+
+    @patch("check_software_raid.dump_raid_info")
+    @patch("check_software_raid.get_md_stat")
+    def test_check_raid_mode_param_with_redundant_space(
+                    self, mock_get_md, mock_dump_raid):
+
+        mock_get_md.return_value = [
+            {"device": "md2", "mode": "raid1"},
+            {"device": "md1", "mode": "raid0"}
+        ]
+
+        check_software_raid.check_raid_mode_test("  raid1  raid0  ")
+        mock_get_md.assert_called_with()
+        mock_dump_raid.assert_called_with(["md2", "md1"])
+
+    @patch("check_software_raid.dump_raid_info")
+    @patch("check_software_raid.get_md_stat")
+    def test_check_raid_mode_is_not_expected(self, mock_get_md, mock_dump_raid):
+
+        mock_get_md.return_value = [
+            {"device": "md2", "mode": "raid1"},
+            {"device": "md1", "mode": "raid0"}
+        ]
+
+        with self.assertRaises(ValueError):
+            check_software_raid.check_raid_mode_test("raid1")
+            mock_get_md.assert_called_with()
+            mock_dump_raid.assert_called_with()
+
+    @patch("subprocess.run")
+    def test_dump_raid_info(self, mock_run):
+
+        check_software_raid.dump_raid_info(["md126", "md127"])
+        self.assertEqual(mock_run.call_count, 2)
+
+
+class TestArgumentParser(unittest.TestCase):
+
+    def test_parser(self):
+        sys.argv = ["check_software_raid.py", "--mode", "raid0 raid1"]
+        args = check_software_raid.register_arguments()
+
+        self.assertEqual(args.mode, "raid0 raid1")

providers/base/tests/test_data/mdstat_intel_rst.txt

@@ -0,0 +1,8 @@
+Personalities : [raid0] [linear] [multipath] [raid1] [raid6] [raid5] [raid4] [raid10]
+md126 : active raid0 sda[1] sdb[0]
+      500113408 blocks super external:/md127/0 64k chunks
+
+md127 : inactive sdb[1](S) sda[0](S)
+      7593 blocks super external:imsm
+
+unused devices: <none>

providers/base/tests/test_data/mdstat_multiple_raid.txt

@@ -0,0 +1,7 @@
+Personalities : [linear] [raid0] [raid1] [raid5]
+read_ahead 1024 sectors
+md2 : active raid1 sde1[1] sdd1[0]
+      17920384 blocks [2/2] [UU]
+
+md1 : active raid0 sdc1[1] sdb1[0]
+      35840768 blocks 64k chunks

providers/base/tests/test_data/mdstat_none_raid.txt

@@ -0,0 +1,2 @@
+Personalities :
+unused devices: <none>

providers/base/units/disk/jobs.pxu

@@ -164,4 +164,21 @@ flags: also-after-suspend
 estimated_duration: 1.0
 user: root
 command: nvme get-feature -f 0x0c -H /dev/{name} | grep '(APSTE).*Enabled' && test -e /sys/class/nvme/{name}/power/pm_qos_latency_tolerance_us
-_summary: Check support for Autonomous Power State Transition on {name}
+_summary: Check support for Autonomous Power State Transition on {name}
+
+plugin: shell
+category_id: com.canonical.plainbox::disk
+id: disk/check-software-raid
+_summary: Validate the configuration of software RAID devices are expected
+_description:
+  Examine the system to detect Software RAID devices are created and the RAID mode are expected
+  the SOFTWARE_RAID_LEVEL variable is needed for this tests.
+  e.g. SOFTWARE_RAID_LEVEL="raid0 raid1 raid5"
+imports: from com.canonical.plainbox import manifest
+requires:
+  executable.name == 'mdadm'
+  manifest.has_md_raid == 'True'
+environ: SOFTWARE_RAID_LEVEL
+user: root
+command: check_software_raid.py --mode "$SOFTWARE_RAID_LEVEL"
+estimated_duration: 10

providers/base/units/disk/manifest.pxu

@@ -0,0 +1,4 @@
+unit: manifest entry
+id: has_md_raid
+_name: Software RAID
+value-type: bool

providers/base/units/disk/packaging.pxu

@@ -12,3 +12,8 @@ Depends: uuid-runtime
 unit: packaging meta-data
 os-id: debian
 Depends: python3-psutil
+
+# This is for disk/check-software-raid
+unit: packaging meta-data
+os-id: debian
+Depends: mdadm

providers/base/units/disk/test-plan.pxu

@@ -18,6 +18,7 @@ include:
     benchmarks/disk/hdparm-read_.*
     benchmarks/disk/hdparm-cache-read_.*
     disk/apste_support_on_.*
+    disk/check-software-raid
 
 id: after-suspend-disk-cert-automated
 unit: test plan
@@ -65,6 +66,7 @@ include:
     disk/stats_.*
     disk/read_performance_.*
     disk/storage_device_.*
+    disk/check-software-raid
 bootstrap_include:
     device