Merge pull request #163 from LedgerHQ/tests/moar

Improving unit test coverage

.codeql-config.yaml

@@ -0,0 +1,3 @@
+paths:
+  - src
+  - doc

.github/workflows/codeql-analysis.yml

@@ -58,3 +58,5 @@ jobs:
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2
+      with:
+        config-file: .codeql-config.yaml

src/ragger/conftest/base_conftest.py

@@ -129,8 +129,8 @@ def prepare_speculos_args(root_pytest_dir: Path, firmware: Firmware, display: bo
     elif len(conf.OPTIONAL.SIDELOADED_APPS) != 0:
         # We are testing a a standalone app that needs libraries: search in SIDELOADED_APPS_DIR
         if conf.OPTIONAL.SIDELOADED_APPS_DIR == "":
-            raise ValueError("Configuration \"SIDELOADED_APPS_DIR\" is mandatory if \
-                             \"SIDELOADED_APPS\" is used")
+            raise ValueError("Configuration \"SIDELOADED_APPS_DIR\" is mandatory if "
+                             "\"SIDELOADED_APPS\" is used")
         libs_dir = Path(project_root_dir / conf.OPTIONAL.SIDELOADED_APPS_DIR)
         # Add "-l Appname:filepath" to Speculos command line for every required lib app
         for coin_name, lib_name in conf.OPTIONAL.SIDELOADED_APPS.items():

src/ragger/navigator/nano_navigator.py

@@ -24,6 +24,8 @@
 class NanoNavigator(Navigator):
 
     def __init__(self, backend: BackendInterface, firmware: Firmware, golden_run: bool = False):
+        if firmware == Firmware.STAX:
+            raise ValueError(f"'{self.__class__.__name__}' does not work on Stax")
         callbacks: Dict[NavInsID, Callable] = {
             NavInsID.WAIT: sleep,
             NavInsID.WAIT_FOR_SCREEN_CHANGE: backend.wait_for_screen_change,

src/ragger/navigator/navigator.py

@@ -25,6 +25,8 @@
 
 from .instruction import NavIns, NavInsID
 
+LAST_SCREEN_UPDATE_TIMEOUT = 2
+
 
 class Navigator(ABC):
 
@@ -418,10 +420,9 @@ def navigate_until_snap(self,
 
             # Make sure there is a screen update after the final action.
             start = time()
-            last_screen_update_timeout = 2
             while self._compare_snap_with_timeout(last_golden_snap, timeout_s=0.5, crop=crop_last):
                 now = time()
-                if (now - start > last_screen_update_timeout):
+                if (now - start > LAST_SCREEN_UPDATE_TIMEOUT):
                     raise TimeoutError(
                         f"Timeout waiting for screen change after last snapshot : {last_golden_snap}"
                     )

src/ragger/navigator/stax_navigator.py

@@ -26,6 +26,8 @@
 class StaxNavigator(Navigator):
 
     def __init__(self, backend: BackendInterface, firmware: Firmware, golden_run: bool = False):
+        if firmware != Firmware.STAX:
+            raise ValueError(f"'{self.__class__.__name__}' only works on Stax")
         screen = FullScreen(backend, firmware)
         callbacks: Dict[NavInsID, Callable] = {
             NavInsID.WAIT: sleep,

src/ragger/utils/misc.py

@@ -17,6 +17,12 @@
 from pathlib import Path
 from ragger.error import ExceptionRAPDU
 
+ERROR_BOLOS_DEVICE_LOCKED = 0x5515
+ERROR_DENIED_BY_USER = 0x5501
+ERROR_APP_NOT_FOUND = 0x6807
+
+ERROR_MSG_DEVICE_LOCKED = "Your device is locked"
+
 
 def find_library_application(base_dir: Path, name: str, device: str) -> Path:
     """
@@ -117,8 +123,8 @@ def get_current_app_name_and_version(backend):
 
         return app_name, version
     except ExceptionRAPDU as e:
-        if e.status == 0x5515:
-            raise ValueError("Your device is locked")
+        if e.status == ERROR_BOLOS_DEVICE_LOCKED:
+            raise ValueError(ERROR_MSG_DEVICE_LOCKED)
         raise e
 
 
@@ -139,8 +145,8 @@ def open_app_from_dashboard(backend, app_name: str):
             p2=0,
             data=app_name.encode())
     except ExceptionRAPDU as e:
-        if e.status == 0x5501:
+        if e.status == ERROR_DENIED_BY_USER:
             raise ValueError("Open app consent denied by the user")
-        elif e.status == 0x6807:
+        elif e.status == ERROR_APP_NOT_FOUND:
             raise ValueError(f"App '{app_name} is not present")
         raise e

tests/unit/backend/test_stub.py

@@ -2,10 +2,24 @@
 
 from ragger.backend import BackendInterface
 from ragger.backend.stub import StubBackend
+from ragger.utils.structs import RAPDU
 
 
 class TestStubBackend(TestCase):
 
+    def setUp(self):
+        self.stub = StubBackend(None)
+
     def test_can_instantiate(self):
-        stub = StubBackend(None)
-        self.assertIsInstance(stub, BackendInterface)
+        self.assertIsInstance(self.stub, BackendInterface)
+
+    def test_emtpy_methods(self):
+        for func in (self.stub.handle_usb_reset, self.stub.send_raw, self.stub.right_click,
+                     self.stub.left_click, self.stub.both_click, self.stub.finger_touch,
+                     self.stub.wait_for_screen_change, self.stub.get_current_screen_content,
+                     self.stub.pause_ticker, self.stub.resume_ticker, self.stub.send_tick):
+            self.assertIsNone(func())
+        for func in (self.stub.receive, self.stub.exchange_raw):
+            self.assertEqual(func(), RAPDU(0x9000, b""))
+        for func in (self.stub.compare_screen_with_snapshot, self.stub.compare_screen_with_text):
+            self.assertTrue(func(None))

tests/unit/conftests/test_base_conftest.py

@@ -0,0 +1,106 @@
+from pathlib import Path
+from unittest import TestCase
+from unittest.mock import patch
+
+from ragger.conftest import base_conftest as bc
+from ragger.firmware import Firmware
+
+from ..helpers import temporary_directory
+
+
+def prepare_base_dir(directory: Path) -> Path:
+    (directory / ".git").mkdir()
+    (directory / "build" / "stax" / "bin").mkdir(parents=True, exist_ok=True)
+    app_path = (directory / "build" / "stax" / "bin" / "app.elf")
+    app_path.touch()
+    return app_path
+
+
+class TestBaseConftest(TestCase):
+
+    def setUp(self):
+        self.seed = "some seed"
+
+    def test_prepare_speculos_args_simplest(self):
+        with temporary_directory() as temp_dir:
+            app_path = prepare_base_dir(temp_dir)
+            result_app, result_args = bc.prepare_speculos_args(temp_dir, Firmware.STAX, False,
+                                                               self.seed)
+            self.assertEqual(result_app, app_path)
+            self.assertEqual(result_args, {"args": ["--seed", self.seed]})
+
+    def test_prepare_speculos_args_simple_with_gui(self):
+        with temporary_directory() as temp_dir:
+            app_path = prepare_base_dir(temp_dir)
+            result_app, result_args = bc.prepare_speculos_args(temp_dir, Firmware.STAX, True,
+                                                               self.seed)
+            self.assertEqual(result_app, app_path)
+            self.assertEqual(result_args, {"args": ["--display", "qt", "--seed", self.seed]})
+
+    def test_prepare_speculos_args_main_as_library(self):
+        with temporary_directory() as temp_dir:
+            app_path = prepare_base_dir(temp_dir)
+            with patch("ragger.conftest.base_conftest.conf.OPTIONAL.LOAD_MAIN_APP_AS_LIBRARY",
+                       True):
+                result_app, result_args = bc.prepare_speculos_args(temp_dir, Firmware.STAX, False,
+                                                                   self.seed)
+                self.assertEqual(result_app, app_path)
+                self.assertEqual(result_args, {"args": [f"-l{app_path}", "--seed", self.seed]})
+
+    def test_prepare_speculos_args_sideloaded_apps_nok_no_dir(self):
+        with temporary_directory() as temp_dir:
+            prepare_base_dir(temp_dir)
+            with patch("ragger.conftest.base_conftest.conf.OPTIONAL.SIDELOADED_APPS",
+                       ["more than 1 elt"]):
+                with self.assertRaises(ValueError):
+                    bc.prepare_speculos_args(temp_dir, Firmware.STAX, False, self.seed)
+
+    def test_prepare_speculos_args_sideloaded_apps_ok(self):
+        lib1_bin, lib1_name, lib2_bin, lib2_name = "lib1", "name1", "lib2", "name2"
+        with temporary_directory() as temp_dir:
+            app_path = prepare_base_dir(temp_dir)
+            sideloaded_apps_dir = temp_dir / "here"
+            sideloaded_apps_dir.mkdir()
+            lib1_path = sideloaded_apps_dir / f"{lib1_bin}_stax.elf"
+            lib2_path = sideloaded_apps_dir / f"{lib2_bin}_stax.elf"
+            lib1_path.touch()
+            lib2_path.touch()
+            with patch("ragger.conftest.base_conftest.conf.OPTIONAL.SIDELOADED_APPS", {
+                    lib1_bin: lib1_name,
+                    lib2_bin: lib2_name
+            }):
+                with patch("ragger.conftest.base_conftest.conf.OPTIONAL.SIDELOADED_APPS_DIR",
+                           sideloaded_apps_dir):
+
+                    result_app, result_args = bc.prepare_speculos_args(
+                        temp_dir, Firmware.STAX, False, self.seed)
+                    self.assertEqual(result_app, app_path)
+                    self.assertEqual(
+                        result_args, {
+                            "args": [
+                                f"-l{lib1_name}:{lib1_path}", f"-l{lib2_name}:{lib2_path}",
+                                "--seed", self.seed
+                            ]
+                        })
+
+    def test_create_backend_nok(self):
+        with self.assertRaises(ValueError):
+            bc.create_backend(None, "does not exist", None, None, None, None)
+
+    def test_create_backend_speculos(self):
+        with patch("ragger.conftest.base_conftest.SpeculosBackend") as backend:
+            with temporary_directory() as temp_dir:
+                prepare_base_dir(temp_dir)
+                result = bc.create_backend(temp_dir, "Speculos", Firmware.STAX, False, None,
+                                           self.seed)
+                self.assertEqual(result, backend())
+
+    def test_create_backend_ledgercomm(self):
+        with patch("ragger.conftest.base_conftest.LedgerWalletBackend") as backend:
+            result = bc.create_backend(None, "ledgerWALLET", Firmware.STAX, False, None, self.seed)
+            self.assertEqual(result, backend())
+
+    def test_create_backend_ledgerwallet(self):
+        with patch("ragger.conftest.base_conftest.LedgerCommBackend") as backend:
+            result = bc.create_backend(None, "LedgerComm", Firmware.STAX, False, None, self.seed)
+            self.assertEqual(result, backend())

tests/unit/helpers.py

@@ -0,0 +1,9 @@
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+
+@contextmanager
+def temporary_directory():
+    with TemporaryDirectory() as dir_path:
+        yield Path(dir_path).resolve()

tests/unit/navigator/test_nano_navigator.py

@@ -0,0 +1,20 @@
+from functools import partial
+from unittest import TestCase
+
+from ragger.navigator.nano_navigator import NanoNavigator
+from ragger.backend import LedgerCommBackend, LedgerWalletBackend, SpeculosBackend
+from ragger.firmware import Firmware
+
+
+class TestNanoNavigator(TestCase):
+
+    def test___init__ok(self):
+        for backend_cls in [
+                partial(SpeculosBackend, "some app"), LedgerCommBackend, LedgerWalletBackend
+        ]:
+            backend = backend_cls(Firmware.NANOS)
+            NanoNavigator(backend, Firmware.NANOS)
+
+    def test___init__nok(self):
+        with self.assertRaises(ValueError):
+            NanoNavigator("whatever", Firmware.STAX)

tests/unit/navigator/test_navigator.py

@@ -1,9 +1,9 @@
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from unittest import TestCase
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
-from ragger.backend import SpeculosBackend
+from ragger.backend import SpeculosBackend, LedgerCommBackend
 from ragger.firmware import Firmware
 from ragger.navigator import Navigator, NavIns, NavInsID
 
@@ -219,3 +219,55 @@ def test_navigate_until_text_and_compare_nok_timeout(self):
 
         with self.assertRaises(TimeoutError):
             self.navigator.navigate_until_text_and_compare(ni, [], "not important", timeout=0)
+
+    def test_navigate_until_snap_not_speculos(self):
+        self.navigator._backend = MagicMock(spec=LedgerCommBackend)
+        self.assertEqual(
+            0,
+            self.navigator.navigate_until_snap(NavInsID.WAIT, NavInsID.WAIT, Path(), Path(), "",
+                                               ""))
+
+    def test_navigate_until_snap_ok(self):
+        self.navigator._backend = MagicMock(spec=SpeculosBackend)
+        self.navigator._check_snaps_dir_path = MagicMock()
+        self.navigator._run_instruction = MagicMock()
+        self.navigator._compare_snap_with_timeout = MagicMock()
+        snapshot_comparisons = (True, True, False)
+        # comparing first snapshot: True
+        # then comparing snapshots until given: True (i.e first snapshot is the expected one)
+        # then waiting for a screen change: False (screen changed)
+        expected_idx = 0
+        # as there is no snapshot between the first image and the last snapshot, the index is 0
+        self.navigator._compare_snap_with_timeout.side_effect = snapshot_comparisons
+        self.assertEqual(
+            expected_idx,
+            self.navigator.navigate_until_snap(NavInsID.WAIT, NavInsID.WAIT, Path(), Path(), "",
+                                               ""))
+
+        snapshot_comparisons = (True, False, False, True, False)
+        # comparing first snapshot: True
+        # then comparing snapshots until given: False, False, True (i.e first two snapshots did not match, but the third is the expected one)
+        # then waiting for a screen change: False (screen changed)
+        expected_idx = 2
+        # as there is 2 snapshots between the first image and the last snapshot, the index is 0
+        self.navigator._compare_snap_with_timeout.side_effect = snapshot_comparisons
+        self.assertEqual(
+            expected_idx,
+            self.navigator.navigate_until_snap(NavInsID.WAIT, NavInsID.WAIT, Path(), Path(), "",
+                                               ""))
+
+    def test_navigate_until_snap_nok_timeout(self):
+        self.navigator._backend = MagicMock(spec=SpeculosBackend)
+        self.navigator._check_snaps_dir_path = MagicMock()
+        self.navigator._run_instruction = MagicMock()
+        self.navigator._compare_snap_with_timeout = MagicMock()
+        self.navigator._compare_snap_with_timeout.return_value = True
+        with patch("ragger.navigator.navigator.LAST_SCREEN_UPDATE_TIMEOUT", 0):
+            with self.assertRaises(TimeoutError):
+                self.navigator.navigate_until_snap(NavInsID.WAIT,
+                                                   NavInsID.WAIT,
+                                                   Path(),
+                                                   Path(),
+                                                   "",
+                                                   "",
+                                                   timeout=0)

tests/unit/navigator/test_stax_navigator.py

@@ -0,0 +1,20 @@
+from functools import partial
+from unittest import TestCase
+
+from ragger.navigator.stax_navigator import StaxNavigator
+from ragger.backend import LedgerCommBackend, LedgerWalletBackend, SpeculosBackend
+from ragger.firmware import Firmware
+
+
+class TestStaxNavigator(TestCase):
+
+    def test___init__ok(self):
+        for backend_cls in [
+                partial(SpeculosBackend, "some app"), LedgerCommBackend, LedgerWalletBackend
+        ]:
+            backend = backend_cls(Firmware.STAX)
+            StaxNavigator(backend, Firmware.STAX)
+
+    def test___init__nok(self):
+        with self.assertRaises(ValueError):
+            StaxNavigator("whatever", Firmware.NANOS)