Add: snaps confinement test (new)

providers/base/bin/snap_confinement_test.py

@@ -17,31 +17,47 @@
 # You should have received a copy of the GNU General Public License
 # along with Checkbox.  If not, see <http://www.gnu.org/licenses/>.
 
+import argparse
 import json
 import logging
+import re
 import sys
 from checkbox_support.snap_utils.snapd import Snapd
 
-features_should_include = [
-    "kernel:caps",
-    "kernel:dbus",
-    "kernel:domain",
-    "kernel:file",
-    "kernel:mount",
-    "kernel:namespaces",
-    "kernel:network",
-    "kernel:ptrace",
-    "kernel:signal",
-    "parser:unsafe",
-]
 
+def test_system_confinement():
+    """
+    Test the system's confinement and sandbox features.
 
-def main():
+    This test checks if the system's confinement is 'strict'. If it
+    is 'strict', the test passes; otherwise, it checks the presence
+    of required features and prints out errors for any missing ones.
+
+    Variables:
+        features_should_include (list): A list of sandbox features that
+            must be present in the 'apparmor' category.
+    Returns:
+        str: A detailed output of the system's confinement and
+             sandbox features in JSON format.
+    """
+    features_should_include = [
+        "kernel:caps",
+        "kernel:dbus",
+        "kernel:domain",
+        "kernel:file",
+        "kernel:mount",
+        "kernel:namespaces",
+        "kernel:network",
+        "kernel:ptrace",
+        "kernel:signal",
+        "parser:unsafe",
+    ]
 
     data = Snapd().get_system_info()
 
     confinement = data["confinement"]
     if confinement == "strict":
+        print("System confinement is \"strict\"")
         print("Test PASS")
         return 0
 
@@ -61,21 +77,110 @@ def main():
                 missing_features.append(feature)
 
     if missing_features:
-        logging.error("Cannot find '{}' in apparmor".format(missing_features))
+        logging.error(
+            "Cannot find '%s' in apparmor", missing_features
+        )
 
     categories_to_check = ["mount", "udev"]
     for category in categories_to_check:
         if category not in sandbox_features:
             logging.error(
-                "Cannot find '{}' in sandbox-features".format(category)
+                "Cannot find '%s' in sandbox-features", category
             )
             break
         for feature in sandbox_features[category]:
             if "cgroup-v2" in feature:
-                logging.error("cgroup({}) must NOT be v2".format(feature))
+                logging.error("cgroup(%s) must NOT be v2", feature)
 
     return sandbox_features_output
 
 
+def test_snaps_confinement():
+    """
+    Test the confinement status of all installed snaps.
+
+    A snap confinement should be 'strict', devmode should be False,
+    and should not have a sideloaded revision starts with 'x'.
+
+    Variables:
+        allowlist_snaps (list): A list of snap names or regex patterns
+            that are exempted from the confinement check. To match the
+            entire snap name, use the pattern "^<snap_name>$". For
+            example, "bugit" matches only "bugit". To match multiple
+            snaps with similar names, use ".*" suffixes. For instance,
+            "checkbox.*" matches all snap names starting with "checkbox".
+            Customize this list to exclude specific snaps from the
+            confinement checks based on their names or patterns.
+    Returns:
+            int: Exit code. 0 if the test passes for all snaps,
+                 otherwise 1.
+    """
+    allowlist_snaps = [
+        r"^bugit$",
+        r"checkbox.*",
+        r"^mir-test-tools$",
+        r"^graphics-test-tools$",
+    ]
+
+    data = Snapd().list()
+    exit_code = 0
+    for snap in data:
+        snap_name = snap.get("name")
+        snap_confinement = snap.get("confinement")
+        snap_devmode = snap.get("devmode")
+        snap_revision = snap.get("revision")
+
+        if snap_name is None:
+            logging.error("Snap 'name' not found in the snap data.")
+            exit_code = 1
+            continue  # Skipping following checks if snap_name not found
+
+        if any(
+            re.match(pattern, snap_name)
+            for pattern in allowlist_snaps
+        ):
+            print("Skipping whitelisted snap: {}".format(snap_name))
+            continue
+
+        if snap_confinement != "strict":
+            exit_code = 1
+            logging.error(
+                "Snap '%s' confinement is expected to be 'strict' "
+                "but got '%s'", snap_name, snap_confinement,
+            )
+
+        if snap_devmode is not False:
+            exit_code = 1
+            logging.error(
+                "Snap '%s' devmode is expected to be False but "
+                "got '%s'", snap_name, snap_devmode,
+            )
+
+        if snap_revision and snap_revision.startswith("x"):
+            exit_code = 1
+            logging.error(
+                "Snap '%s' has sideloaded revision '%s', which "
+                "is not allowed", snap_name, snap_revision,
+            )
+        elif snap_revision is None:
+            exit_code = 1
+            logging.error(
+                "'revision' not found in snap '%s'", snap_name,
+            )
+    return exit_code
+
+
+def main():
+    logging.basicConfig(format='%(levelname)s: %(message)s')
+    sub_commands = {
+        "system": test_system_confinement,
+        "snaps": test_snaps_confinement,
+    }
+    parser = argparse.ArgumentParser()
+    parser.add_argument("subcommand", type=str, choices=sub_commands)
+    args = parser.parse_args()
+    return sub_commands[args.subcommand]()
+
+
 if __name__ == "__main__":
     sys.exit(main())

providers/base/units/snapd/snapd.pxu

@@ -274,13 +274,23 @@ category_id: snapd
 estimated_duration: 1.0
 flags: preserve-locale
 
-id: snappy/test-snap-confinement-mode
-_summary: Test if the snap confinement mode is strict
+id: snappy/test-system-confinement
+_summary: Test if the system confinement is strict
 _purpose:
- Test if the snap confinement mode is `strict`
- If not, list the missing features
+  Test if the system confinement is "strict"
+  If not, list the missing features
 plugin: shell
-command: snap_confinement_test.py
-user: root
+command: snap_confinement_test.py system
+category_id: snapd
+estimated_duration: 2s
+
+id: snappy/test-snaps-confinement
+_summary: Test all the snaps' confinement
+_purpose:
+  Test all the snaps' confinement, devmode, revision.
+  Make sure the confinement is "strict", devmode is "False",
+  and revision should not be sideloded.
+plugin: shell
+command: snap_confinement_test.py snaps
 category_id: snapd
-estimated_duration: 1s
+estimated_duration: 3s

providers/base/units/snapd/test-plan.pxu

@@ -68,7 +68,8 @@ include:
     snappy/test-store-install-beta
     snappy/test-store-install-edge
     snappy/test-store-config-.*
-    snappy/test-snap-confinement-mode
+    snappy/test-system-confinement
+    snappy/test-snaps-confinement
 mandatory_include:
     snap
 bootstrap_include:

providers/certification-client/units/client-cert-desktop-18-04.pxu

@@ -109,6 +109,7 @@ nested_part:
     networking-cert-automated
     optical-cert-automated
     power-management-precheck-cert-automated
+    snappy-snap-automated
     touchpad-cert-automated
     touchscreen-cert-automated
     usb-cert-automated

providers/certification-client/units/client-cert-desktop-20-04.pxu

@@ -112,6 +112,7 @@ nested_part:
     networking-cert-automated
     optical-cert-automated
     power-management-precheck-cert-automated
+    snappy-snap-automated
     touchpad-cert-automated
     touchscreen-cert-automated
     usb-cert-automated

providers/certification-client/units/client-cert-desktop-22-04.pxu

@@ -42,6 +42,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     touchpad-cert-manual
     touchscreen-cert-manual
     usb-cert-manual

providers/certification-client/units/client-cert-odm-desktop-18-04.pxu

@@ -26,6 +26,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     # touchpad-cert-manual
     # touchscreen-cert-manual
     usb-cert-manual

providers/certification-client/units/client-cert-odm-desktop-20-04.pxu

@@ -27,6 +27,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     # touchpad-cert-manual
     # touchscreen-cert-manual
     usb-cert-manual

providers/certification-client/units/client-cert-odm-desktop-22-04.pxu

@@ -27,6 +27,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     # touchpad-cert-manual
     # touchscreen-cert-manual
     usb-cert-manual