Implement V4L2 compliance parser (New)

checkbox-support/checkbox_support/parsers/tests/test_v4l2_compliance.py

@@ -0,0 +1,64 @@
+import subprocess as sp
+from checkbox_support.parsers.v4l2_compliance import parse_v4l2_compliance
+import unittest as ut
+from unittest.mock import patch, MagicMock
+from pkg_resources import resource_filename
+
+
+def read_file_as_str(name: str):
+    resource = "parsers/tests/v4l2_compliance_data/{}.txt".format(name)
+    filename = resource_filename("checkbox_support", resource)
+    with open(filename) as f:
+        return f.read()
+
+
+class TestV4L2ComplianceParser(ut.TestCase):
+
+    @patch("subprocess.run")
+    def test_happy_path(self, mock_run: MagicMock):
+        ok_input = read_file_as_str("22_04_success")
+        mock_run.return_value = sp.CompletedProcess(
+            [], 1, stdout=ok_input, stderr=""
+        )
+        summary, detail = parse_v4l2_compliance()
+        self.assertDictEqual(
+            {
+                "device_name": "uvcvideo device /dev/video0",
+                "total": 46,
+                "succeeded": 43,
+                "failed": 3,
+                "warnings": 1,
+            },
+            summary,
+        )
+        expected_failures = read_file_as_str("output_failed_ioctls_1")
+        for ioctl_request in expected_failures.splitlines():
+            self.assertIn(ioctl_request.strip(), detail["failed"])
+
+    @patch("subprocess.run")
+    def test_unparsable(self, mock_run: MagicMock):
+        bad_input = "askdjhasjkdhlakbbeqmnwbeqmvykudsuchab,b1231"
+        mock_run.return_value = sp.CompletedProcess(
+            [], 1, stdout=bad_input, stderr=""
+        )
+
+        self.assertRaises(AssertionError, parse_v4l2_compliance)
+
+    @patch("subprocess.run")
+    def test_unopenable_device(self, mock_run: MagicMock):
+        err_messages = [
+            # 16.04 18.04: found this message in VMs without camera pass through
+            "Failed to open device /dev/video0: No such file or directory"
+            # 20.04: found this msg in VMs without camera pass through
+            # 22.04, 24.04: found this message if we disable camera in BIOS
+            "Cannot open device /dev/video0, exiting."
+        ]
+        for err_msg in err_messages:
+            mock_run.return_value = sp.CompletedProcess(
+                [], 1, stdout="", stderr=err_msg
+            )
+            self.assertRaises(FileNotFoundError, parse_v4l2_compliance)
+
+
+if __name__ == "__main__":
+    ut.main()

checkbox-support/checkbox_support/parsers/tests/v4l2_compliance_data/22_04_success.txt

@@ -0,0 +1,112 @@
+v4l2-compliance 1.22.1, 64 bits, 64-bit time_t
+
+Compliance test for uvcvideo device /dev/video0:
+
+Driver Info:
+	Driver name      : uvcvideo
+	Card type        : Integrated_Webcam_HD: Integrate
+	Bus info         : usb-0000:00:14.0-9
+	Driver version   : 6.8.12
+	Capabilities     : 0x84a00001
+		Video Capture
+		Metadata Capture
+		Streaming
+		Extended Pix Format
+		Device Capabilities
+	Device Caps      : 0x04200001
+		Video Capture
+		Streaming
+		Extended Pix Format
+Media Driver Info:
+	Driver name      : uvcvideo
+	Model            : Integrated_Webcam_HD: Integrate
+	Serial           : 200901010001
+	Bus info         : usb-0000:00:14.0-9
+	Media version    : 6.8.12
+	Hardware revision: 0x00009628 (38440)
+	Driver version   : 6.8.12
+Interface Info:
+	ID               : 0x03000002
+	Type             : V4L Video
+Entity Info:
+	ID               : 0x00000001 (1)
+	Name             : Integrated_Webcam_HD: Integrate
+	Function         : V4L2 I/O
+	Flags            : default
+	Pad 0x01000007   : 0: Sink
+	  Link 0x02000013: from remote pad 0x100000a of entity 'Extension 4' (Video Pixel Formatter): Data, Enabled, Immutable
+
+Required ioctls:
+	test MC information (see 'Media Driver Info' above): OK
+	test VIDIOC_QUERYCAP: OK
+	test invalid ioctls: OK
+
+Allow for multiple opens:
+	test second /dev/video0 open: OK
+	test VIDIOC_QUERYCAP: OK
+	test VIDIOC_G/S_PRIORITY: OK
+	test for unlimited opens: OK
+
+Debug ioctls:
+	test VIDIOC_DBG_G/S_REGISTER: OK (Not Supported)
+	test VIDIOC_LOG_STATUS: OK (Not Supported)
+
+Input ioctls:
+	test VIDIOC_G/S_TUNER/ENUM_FREQ_BANDS: OK (Not Supported)
+	test VIDIOC_G/S_FREQUENCY: OK (Not Supported)
+	test VIDIOC_S_HW_FREQ_SEEK: OK (Not Supported)
+	test VIDIOC_ENUMAUDIO: OK (Not Supported)
+	test VIDIOC_G/S/ENUMINPUT: OK
+	test VIDIOC_G/S_AUDIO: OK (Not Supported)
+	Inputs: 1 Audio Inputs: 0 Tuners: 0
+
+Output ioctls:
+	test VIDIOC_G/S_MODULATOR: OK (Not Supported)
+	test VIDIOC_G/S_FREQUENCY: OK (Not Supported)
+	test VIDIOC_ENUMAUDOUT: OK (Not Supported)
+	test VIDIOC_G/S/ENUMOUTPUT: OK (Not Supported)
+	test VIDIOC_G/S_AUDOUT: OK (Not Supported)
+	Outputs: 0 Audio Outputs: 0 Modulators: 0
+
+Input/Output configuration ioctls:
+	test VIDIOC_ENUM/G/S/QUERY_STD: OK (Not Supported)
+	test VIDIOC_ENUM/G/S/QUERY_DV_TIMINGS: OK (Not Supported)
+	test VIDIOC_DV_TIMINGS_CAP: OK (Not Supported)
+	test VIDIOC_G/S_EDID: OK (Not Supported)
+
+Control ioctls (Input 0):
+	test VIDIOC_QUERY_EXT_CTRL/QUERYMENU: OK
+	test VIDIOC_QUERYCTRL: OK
+		fail: v4l2-test-controls.cpp(489): s_ctrl returned an error (13)
+	test VIDIOC_G/S_CTRL: FAIL
+		fail: v4l2-test-controls.cpp(736): s_ext_ctrls returned an error (13)
+	test VIDIOC_G/S/TRY_EXT_CTRLS: FAIL
+	test VIDIOC_(UN)SUBSCRIBE_EVENT/DQEVENT: OK
+	test VIDIOC_G/S_JPEGCOMP: OK (Not Supported)
+	Standard Controls: 16 Private Controls: 0
+
+Format ioctls (Input 0):
+	test VIDIOC_ENUM_FMT/FRAMESIZES/FRAMEINTERVALS: OK
+	test VIDIOC_G/S_PARM: OK
+	test VIDIOC_G_FBUF: OK (Not Supported)
+	test VIDIOC_G_FMT: OK
+	test VIDIOC_TRY_FMT: OK
+		warn: v4l2-test-formats.cpp(1036): Could not set fmt2
+	test VIDIOC_S_FMT: OK
+	test VIDIOC_G_SLICED_VBI_CAP: OK (Not Supported)
+	test Cropping: OK (Not Supported)
+	test Composing: OK (Not Supported)
+	test Scaling: OK (Not Supported)
+
+Codec ioctls (Input 0):
+	test VIDIOC_(TRY_)ENCODER_CMD: OK (Not Supported)
+	test VIDIOC_G_ENC_INDEX: OK (Not Supported)
+	test VIDIOC_(TRY_)DECODER_CMD: OK (Not Supported)
+
+Buffer ioctls (Input 0):
+		fail: v4l2-test-buffers.cpp(703): check_0(crbufs.reserved, sizeof(crbufs.reserved))
+	test VIDIOC_REQBUFS/CREATE_BUFS/QUERYBUF: FAIL
+	test VIDIOC_EXPBUF: OK
+	test Requests: OK (Not Supported)
+
+Total for uvcvideo device /dev/video0: 46, Succeeded: 43, Failed: 3, Warnings: 1

checkbox-support/checkbox_support/parsers/tests/v4l2_compliance_data/output_failed_ioctls_1.txt

@@ -0,0 +1,8 @@
+VIDIOC_REQBUFS
+VIDIOC_CREATE_BUFS
+VIDIOC_QUERYBUF
+VIDIOC_G_EXT_CTRLS
+VIDIOC_S_EXT_CTRLS
+VIDIOC_TRY_EXT_CTRLS
+VIDIOC_G_CTRL
+VIDIOC_S_CTRL

checkbox-support/checkbox_support/parsers/v4l2_compliance.py

@@ -0,0 +1,196 @@
+#! /usr/bin/python3
+
+import re
+from shutil import which
+import subprocess as sp
+import typing as T
+
+
+# Not going to try to parse these test names since the
+# ioctls in the test are not necessarily described in its name
+TEST_NAME_TO_IOCTL_MAP = {
+    "VIDIOC_QUERYCAP": ["VIDIOC_QUERYCAP"],
+    "VIDIOC_G/S_PRIORITY": ["VIDIOC_G_PRIORITY", "VIDIOC_S_PRIORITY"],
+    "VIDIOC_DBG_G/S_REGISTER": [
+        "VIDIOC_DBG_G_REGISTER",
+        "VIDIOC_DBG_S_REGISTER",
+    ],
+    "VIDIOC_LOG_STATUS": ["VIDIOC_LOG_STATUS"],
+    "VIDIOC_G/S_TUNER/ENUM_FREQ_BANDS": [
+        "VIDIOC_G_TUNER",
+        "VIDIOC_S_TUNER",
+        "VIDIO_ENUM_FREQ_BANDS",
+    ],
+    "VIDIOC_G/S_FREQUENCY": ["VIDIOC_G_FREQUENCY", "VIDIOC_S_FREQUENCY"],
+    "VIDIOC_S_HW_FREQ_SEEK": ["VIDIOC_S_HW_FREQ_SEEK"],
+    "VIDIOC_ENUMAUDIO": ["VIDIOC_ENUMAUDIO"],
+    "VIDIOC_G/S/ENUMINPUT": [
+        "VIDIOC_G_SELECTION",
+        "VIDIOC_ENUMINPUT",
+        "VIDIOC_S_INPUT",
+    ],
+    "VIDIOC_G/S_AUDIO": ["VIDIOC_G_AUDIO", "VIDIOC_S_AUDIO"],
+    "VIDIOC_G/S_MODULATOR": ["VIDIOC_G_MODULATOR", "VIDIOC_S_MODULATOR"],
+    "VIDIOC_G/S_FREQUENCY": ["VIDIOC_G_FREQUENCY", "VIDIOC_S_FREQUENCY"],
+    "VIDIOC_ENUMAUDOUT": ["VIDIOC_ENUMAUDOUT"],
+    "VIDIOC_G/S/ENUMOUTPUT": [
+        "VIDIOC_G_OUTPUT",
+        "VIDIOC_S_OUTPUT",
+        "VIDIOC_ENUMOUTPUT",
+    ],
+    "VIDIOC_G/S_AUDOUT": ["VIDIOC_G_AUDOUT", "VIDIOC_S_AUDOUT"],
+    "VIDIOC_ENUM/G/S/QUERY_STD": [
+        "VIDIOC_ENUMSTD",
+        "VIDIOC_G_STD",
+        "VIDIOC_S_STD",
+        "VIDIOC_QUERYSTD",
+    ],
+    "VIDIOC_ENUM/G/S/QUERY_DV_TIMINGS": [
+        "VIDIOC_G_DV_TIMINGS",
+        "VIDIOC_ENUM_DV_TIMINGS",
+        "VIDIOC_QUERY_DV_TIMINGS",
+    ],
+    "VIDIOC_DV_TIMINGS_CAP": ["VIDIOC_DV_TIMINGS_CAP"],
+    "VIDIOC_G/S_EDID": ["VIDIOC_G_EDID", "VIDIOC_S_EDID"],
+    "VIDIOC_QUERY_EXT_CTRL/QUERYMENU": [
+        "VIDIOC_QUERYMENU",
+        "VIDIOC_QUERY_EXT_CTRL",
+    ],
+    "VIDIOC_QUERYCTRL": ["VIDIOC_QUERYCTRL"],
+    "VIDIOC_G/S_CTRL": ["VIDIOC_G_CTRL", "VIDIOC_S_CTRL"],
+    "VIDIOC_G/S/TRY_EXT_CTRLS": [
+        "VIDIOC_G_EXT_CTRLS",
+        "VIDIOC_S_EXT_CTRLS",
+        "VIDIOC_TRY_EXT_CTRLS",
+    ],
+    "VIDIOC_(UN)SUBSCRIBE_EVENT/DQEVENT": [
+        "VIDIOC_SUBSCRIBE_EVENT",
+        "VIDIOC_UNSUBSCRIBE_EVENT",
+    ],
+    "VIDIOC_G/S_JPEGCOMP": ["VIDIOC_G_JPEGCOMP", "VIDIOC_S_JPEGCOMP"],
+    "VIDIOC_ENUM_FMT/FRAMESIZES/FRAMEINTERVALS": [
+        "VIDIOC_ENUM_FMT",
+        "VIDIOC_ENUM_FRAMEINTERVALS",
+        "VIDIOC_ENUM_FRAMESIZES",
+    ],
+    "VIDIOC_G/S_PARM": ["VIDIOC_G_PARM", "VIDIOC_S_PARM"],
+    "VIDIOC_G_FBUF": ["VIDIOC_G_FBUF"],
+    "VIDIOC_G_FMT": ["VIDIOC_G_FMT"],
+    "VIDIOC_TRY_FMT": ["VIDIOC_TRY_FMT"],
+    "VIDIOC_S_FMT": ["VIDIOC_S_FMT"],
+    "VIDIOC_G_SLICED_VBI_CAP": ["VIDIOC_G_SLICED_VBI_CAP"],
+    "VIDIOC_(TRY_)ENCODER_CMD": [
+        "VIDIOC_ENCODER_CMD",
+        "VIDIOC_TRY_ENCODER_CMD",
+    ],
+    "VIDIOC_G_ENC_INDEX": ["VIDIOC_G_ENC_INDEX"],
+    "VIDIOC_(TRY_)DECODER_CMD": [
+        "VIDIOC_DECODER_CMD",
+        "VIDIOC_TRY_DECODER_CMD",
+    ],
+    "VIDIOC_REQBUFS/CREATE_BUFS/QUERYBUF": [
+        "VIDIOC_REQBUFS",
+        "VIDIOC_CREATE_BUFS",
+        "VIDIOC_QUERYBUF",
+    ],
+    "VIDIOC_EXPBUF": ["VIDIOC_EXPBUF"],
+}
+
+
+# see the summary dict literal for actual keys
+Summary = T.Dict[str, T.Union[int, str]]
+# see the details dict literal for actual keys
+Details = T.Dict[str, T.List[str]]
+
+
+def get_test_name_from_line(line: str) -> T.Tuple[str, bool]:
+    assert line.startswith("test"), "This line doesn't describe a test output"
+    test_name = line.split("test ", maxsplit=1)[1].split(": ", maxsplit=1)[0]
+    return test_name, test_name.startswith("VIDIOC")
+
+
+def parse_v4l2_compliance(
+    device: T.Optional[str] = None,
+) -> T.Tuple[Summary, Details]:
+    """Parses the output of v4l2-compliance
+
+    :param device: which device to test, defaults to "/dev/video0",
+    it can also be an integer. See v4l2-compliance -h
+    :type device: T.Union[int, str], optional
+    :return: 2 dictionaries (summary, details).
+    NOTE: summary comes from directly parsing the numbers in the last line of
+    v4l2-compliance and it does **NOT** match the array sizes in Details
+    since we map the test names to actual ioctls.
+
+    :rtype: T.Tuple[Summary, Details]
+    """
+
+    out = sp.run(
+        [
+            "v4l2-compliance",
+            *(["-d", str(device)] if device else []),
+            "-C",
+            "never",
+        ],
+        universal_newlines=True,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE,
+    )  # can't really depend on the return code here
+    # since any failure => return code 1
+
+    error_prefixes = ("Failed to open", "Cannot open device")
+    if any(out.stderr.startswith(prefix) for prefix in error_prefixes):
+        # can't open the device
+        raise FileNotFoundError(out.stderr)
+
+    lines = []  # type: list[str]
+    for line in out.stdout.splitlines():
+        clean_line = line.strip()
+        if clean_line != "":
+            lines.append(clean_line)
+
+    pattern = (
+        r"Total for (.*): (.*), Succeeded: (.*), Failed: (.*), Warnings: (.*)"
+    )
+    match_output = re.match(pattern, lines[-1])
+
+    summary = {}
+    if match_output:
+        summary = {
+            "device_name": match_output.group(1),
+            "total": int(match_output.group(2)),
+            "succeeded": int(match_output.group(3)),
+            "failed": int(match_output.group(4)),
+            "warnings": int(match_output.group(5)),
+        }
+
+    assert summary != {}, (
+        "There's no summary line in v4l2-compliance's output. "
+        "Output might be corrupted."
+    )
+
+    details = {
+        "succeeded": [],
+        "failed": [],
+        "not_supported": [],
+    }  # type: dict[str, list[str]]
+
+    for line in lines:
+        if line.endswith(": OK"):
+            name, is_ioctl_name = get_test_name_from_line(line)
+            if is_ioctl_name:
+                # ignore unknown test names, just don't append
+                for ioctl_name in TEST_NAME_TO_IOCTL_MAP.get(name, []):
+                    details["succeeded"].append(ioctl_name)
+        elif line.endswith(": OK (Not Supported)"):
+            name, is_ioctl_name = get_test_name_from_line(line)
+            if is_ioctl_name:
+                for ioctl_name in TEST_NAME_TO_IOCTL_MAP.get(name, []):
+                    details["not_supported"].append(ioctl_name)
+        elif line.endswith(": FAIL"):
+            name, is_ioctl_name = get_test_name_from_line(line)
+            if is_ioctl_name:
+                for ioctl_name in TEST_NAME_TO_IOCTL_MAP.get(name, []):
+                    details["failed"].append(ioctl_name)
+
+    return summary, details