Add: snaps confinement test (new) (#663)

* Change: re-structured the system confinement test

The original snappy/test-snap-confinement-mode is actually testing
the system's confinement whether it is "strict". If it is, test
passes. Otherwise, the missing feature to make system in strict
confinement will be listed.

This commit renamed the job ID and purpose to make it more clear.
Also, restructured the python script to put the original test into
SystemConfinement class as a test and added docstring to make it
easier to understand and maintain.

* Change: replace logging string from format() to %

* Add: snaps confinement tests

Add SnapsConfinement class in snap_confinement_test.py to check
all the installed snaps' confinement, devmode, and revision.

* Add: snappy-snap-automated to desktop test plans

* Change: apply suggestions from kissiel

* Change: Flatten simple classes to test functions

providers/base/bin/snap_confinement_test.py

@@ -17,31 +17,47 @@
 # You should have received a copy of the GNU General Public License
 # along with Checkbox.  If not, see <http://www.gnu.org/licenses/>.
 
+import argparse
 import json
 import logging
+import re
 import sys
 from checkbox_support.snap_utils.snapd import Snapd
 
-features_should_include = [
-    "kernel:caps",
-    "kernel:dbus",
-    "kernel:domain",
-    "kernel:file",
-    "kernel:mount",
-    "kernel:namespaces",
-    "kernel:network",
-    "kernel:ptrace",
-    "kernel:signal",
-    "parser:unsafe",
-]
 
+def test_system_confinement():
+    """
+    Test the system's confinement and sandbox features.
 
-def main():
+    This test checks if the system's confinement is 'strict'. If it
+    is 'strict', the test passes; otherwise, it checks the presence
+    of required features and prints out errors for any missing ones.
+
+    Variables:
+        features_should_include (list): A list of sandbox features that
+            must be present in the 'apparmor' category.
+    Returns:
+        str: A detailed output of the system's confinement and
+             sandbox features in JSON format.
+    """
+    features_should_include = [
+        "kernel:caps",
+        "kernel:dbus",
+        "kernel:domain",
+        "kernel:file",
+        "kernel:mount",
+        "kernel:namespaces",
+        "kernel:network",
+        "kernel:ptrace",
+        "kernel:signal",
+        "parser:unsafe",
+    ]
 
     data = Snapd().get_system_info()
 
     confinement = data["confinement"]
     if confinement == "strict":
+        print("System confinement is \"strict\"")
         print("Test PASS")
         return 0
 
@@ -61,21 +77,110 @@ def main():
                 missing_features.append(feature)
 
     if missing_features:
-        logging.error("Cannot find '{}' in apparmor".format(missing_features))
+        logging.error(
+            "Cannot find '%s' in apparmor", missing_features
+        )
 
     categories_to_check = ["mount", "udev"]
     for category in categories_to_check:
         if category not in sandbox_features:
             logging.error(
-                "Cannot find '{}' in sandbox-features".format(category)
+                "Cannot find '%s' in sandbox-features", category
             )
             break
         for feature in sandbox_features[category]:
             if "cgroup-v2" in feature:
-                logging.error("cgroup({}) must NOT be v2".format(feature))
+                logging.error("cgroup(%s) must NOT be v2", feature)
 
     return sandbox_features_output
 
 
+def test_snaps_confinement():
+    """
+    Test the confinement status of all installed snaps.
+
+    A snap confinement should be 'strict', devmode should be False,
+    and should not have a sideloaded revision starts with 'x'.
+
+    Variables:
+        allowlist_snaps (list): A list of snap names or regex patterns
+            that are exempted from the confinement check. To match the
+            entire snap name, use the pattern "^<snap_name>$". For
+            example, "bugit" matches only "bugit". To match multiple
+            snaps with similar names, use ".*" suffixes. For instance,
+            "checkbox.*" matches all snap names starting with "checkbox".
+            Customize this list to exclude specific snaps from the
+            confinement checks based on their names or patterns.
+    Returns:
+            int: Exit code. 0 if the test passes for all snaps,
+                 otherwise 1.
+    """
+    allowlist_snaps = [
+        r"^bugit$",
+        r"checkbox.*",
+        r"^mir-test-tools$",
+        r"^graphics-test-tools$",
+    ]
+
+    data = Snapd().list()
+    exit_code = 0
+    for snap in data:
+        snap_name = snap.get("name")
+        snap_confinement = snap.get("confinement")
+        snap_devmode = snap.get("devmode")
+        snap_revision = snap.get("revision")
+
+        if snap_name is None:
+            logging.error("Snap 'name' not found in the snap data.")
+            exit_code = 1
+            continue  # Skipping following checks if snap_name not found
+
+        if any(
+            re.match(pattern, snap_name)
+            for pattern in allowlist_snaps
+        ):
+            print("Skipping whitelisted snap: {}".format(snap_name))
+            continue
+
+        if snap_confinement != "strict":
+            exit_code = 1
+            logging.error(
+                "Snap '%s' confinement is expected to be 'strict' "
+                "but got '%s'", snap_name, snap_confinement,
+            )
+
+        if snap_devmode is not False:
+            exit_code = 1
+            logging.error(
+                "Snap '%s' devmode is expected to be False but "
+                "got '%s'", snap_name, snap_devmode,
+            )
+
+        if snap_revision and snap_revision.startswith("x"):
+            exit_code = 1
+            logging.error(
+                "Snap '%s' has sideloaded revision '%s', which "
+                "is not allowed", snap_name, snap_revision,
+            )
+        elif snap_revision is None:
+            exit_code = 1
+            logging.error(
+                "'revision' not found in snap '%s'", snap_name,
+            )
+    return exit_code
+
+
+def main():
+    logging.basicConfig(format='%(levelname)s: %(message)s')
+    sub_commands = {
+        "system": test_system_confinement,
+        "snaps": test_snaps_confinement,
+    }
+    parser = argparse.ArgumentParser()
+    parser.add_argument("subcommand", type=str, choices=sub_commands)
+    args = parser.parse_args()
+    return sub_commands[args.subcommand]()
+
+
 if __name__ == "__main__":
     sys.exit(main())

providers/base/units/snapd/snapd.pxu

@@ -274,13 +274,23 @@ category_id: snapd
 estimated_duration: 1.0
 flags: preserve-locale
 
-id: snappy/test-snap-confinement-mode
-_summary: Test if the snap confinement mode is strict
+id: snappy/test-system-confinement
+_summary: Test if the system confinement is strict
 _purpose:
- Test if the snap confinement mode is `strict`
- If not, list the missing features
+  Test if the system confinement is "strict"
+  If not, list the missing features
 plugin: shell
-command: snap_confinement_test.py
-user: root
+command: snap_confinement_test.py system
+category_id: snapd
+estimated_duration: 2s
+
+id: snappy/test-snaps-confinement
+_summary: Test all the snaps' confinement
+_purpose:
+  Test all the snaps' confinement, devmode, revision.
+  Make sure the confinement is "strict", devmode is "False",
+  and revision should not be sideloded.
+plugin: shell
+command: snap_confinement_test.py snaps
 category_id: snapd
-estimated_duration: 1s
+estimated_duration: 3s

providers/base/units/snapd/test-plan.pxu

@@ -68,7 +68,8 @@ include:
     snappy/test-store-install-beta
     snappy/test-store-install-edge
     snappy/test-store-config-.*
-    snappy/test-snap-confinement-mode
+    snappy/test-system-confinement
+    snappy/test-snaps-confinement
 mandatory_include:
     snap
 bootstrap_include:

providers/certification-client/units/client-cert-desktop-18-04.pxu

@@ -109,6 +109,7 @@ nested_part:
     networking-cert-automated
     optical-cert-automated
     power-management-precheck-cert-automated
+    snappy-snap-automated
     touchpad-cert-automated
     touchscreen-cert-automated
     usb-cert-automated

providers/certification-client/units/client-cert-desktop-20-04.pxu

@@ -112,6 +112,7 @@ nested_part:
     networking-cert-automated
     optical-cert-automated
     power-management-precheck-cert-automated
+    snappy-snap-automated
     touchpad-cert-automated
     touchscreen-cert-automated
     usb-cert-automated

providers/certification-client/units/client-cert-desktop-22-04.pxu

@@ -42,6 +42,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     touchpad-cert-manual
     touchscreen-cert-manual
     usb-cert-manual

providers/certification-client/units/client-cert-odm-desktop-18-04.pxu

@@ -26,6 +26,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     # touchpad-cert-manual
     # touchscreen-cert-manual
     usb-cert-manual

providers/certification-client/units/client-cert-odm-desktop-20-04.pxu

@@ -27,6 +27,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     # touchpad-cert-manual
     # touchscreen-cert-manual
     usb-cert-manual

providers/certification-client/units/client-cert-odm-desktop-22-04.pxu

@@ -27,6 +27,7 @@ nested_part:
     networking-cert-manual
     optical-cert-manual
     power-management-precheck-cert-manual
+    snappy-snap-automated
     # touchpad-cert-manual
     # touchscreen-cert-manual
     usb-cert-manual