Merge pull request #51 from citruz/improve-filtering

improved reliability of packet prefiltering

README.rst

@@ -126,6 +126,8 @@ Changelog
 ---------
 Beacontools follows the `semantic versioning <https://semver.org/>`__ scheme.
 
+* 2.0.2
+    * Improved prefiltering of packets, fixes #48
 * 2.0.1
     * Removed (unused) rfu field from the Eddystone UID packet, fixes #39
 * 2.0.0

beacontools/scanner.py

@@ -1,21 +1,26 @@
 """Classes responsible for Beacon scanning."""
-import threading
-import struct
 import logging
+import struct
+import threading
 from importlib import import_module
 
-from .parser import parse_packet
-from .utils import bt_addr_to_string
-from .packet_types import (EddystoneUIDFrame, EddystoneURLFrame,
-                           EddystoneEncryptedTLMFrame, EddystoneTLMFrame,
-                           EddystoneEIDFrame,)
-from .device_filters import BtAddrFilter, DeviceFilter
-from .utils import is_packet_type, is_one_of, to_int, bin_to_int, get_mode
-from .const import (ScannerMode, ScanType, ScanFilter, BluetoothAddressType,
-                    LE_META_EVENT, OGF_LE_CTL, OCF_LE_SET_SCAN_ENABLE,
-                    OCF_LE_SET_SCAN_PARAMETERS, EVT_LE_ADVERTISING_REPORT,
-                    MS_FRACTION_DIVIDER, EXPOSURE_NOTIFICATION_UUID)
+from ahocorapy.keywordtree import KeywordTree
 
+from .const import (CJ_MANUFACTURER_ID, EDDYSTONE_UUID,
+                    ESTIMOTE_MANUFACTURER_ID, ESTIMOTE_UUID,
+                    EVT_LE_ADVERTISING_REPORT, EXPOSURE_NOTIFICATION_UUID,
+                    IBEACON_MANUFACTURER_ID, IBEACON_PROXIMITY_TYPE,
+                    LE_META_EVENT, MANUFACTURER_SPECIFIC_DATA_TYPE,
+                    MS_FRACTION_DIVIDER, OCF_LE_SET_SCAN_ENABLE,
+                    OCF_LE_SET_SCAN_PARAMETERS, OGF_LE_CTL,
+                    BluetoothAddressType, ScanFilter, ScannerMode, ScanType)
+from .device_filters import BtAddrFilter, DeviceFilter
+from .packet_types import (EddystoneEIDFrame, EddystoneEncryptedTLMFrame,
+                           EddystoneTLMFrame, EddystoneUIDFrame,
+                           EddystoneURLFrame)
+from .parser import parse_packet
+from .utils import (bin_to_int, bt_addr_to_string, get_mode, is_one_of,
+                    is_packet_type, to_int)
 
 _LOGGER = logging.getLogger(__name__)
 _LOGGER.setLevel(logging.DEBUG)
@@ -91,6 +96,22 @@ def __init__(self, callback, bt_device_id, device_filter, packet_filter, scan_pa
         # parameters to pass to bt device
         self.scan_parameters = scan_parameters
 
+        # construct an aho-corasick search tree for efficient prefiltering
+        service_uuid_prefix = b"\x03\x03"
+        self.kwtree = KeywordTree()
+        if self.mode & ScannerMode.MODE_IBEACON:
+            self.kwtree.add(bytes([MANUFACTURER_SPECIFIC_DATA_TYPE]) + IBEACON_MANUFACTURER_ID + IBEACON_PROXIMITY_TYPE)
+        if self.mode & ScannerMode.MODE_EDDYSTONE:
+            self.kwtree.add(service_uuid_prefix + EDDYSTONE_UUID)
+        if self.mode & ScannerMode.MODE_ESTIMOTE:
+            self.kwtree.add(service_uuid_prefix + ESTIMOTE_UUID)
+            self.kwtree.add(bytes([MANUFACTURER_SPECIFIC_DATA_TYPE]) + ESTIMOTE_MANUFACTURER_ID)
+        if self.mode & ScannerMode.MODE_CJMONITOR:
+            self.kwtree.add(bytes([MANUFACTURER_SPECIFIC_DATA_TYPE]) + CJ_MANUFACTURER_ID)
+        if self.mode & ScannerMode.MODE_EXPOSURE_NOTIFICATION:
+            self.kwtree.add(service_uuid_prefix + EXPOSURE_NOTIFICATION_UUID)
+        self.kwtree.finalize()
+
     def run(self):
         """Continously scan for BLE advertisements."""
         self.socket = self.backend.open_dev(self.bt_device_id)
@@ -161,22 +182,16 @@ def toggle_scan(self, enable, filter_duplicates=False):
 
     def process_packet(self, pkt):
         """Parse the packet and call callback if one of the filters matches."""
-
+        payload = pkt[14:-1]
         # check if this could be a valid packet before parsing
         # this reduces the CPU load significantly
-        if not ( \
-            ((self.mode & ScannerMode.MODE_IBEACON) and (pkt[19:23] == b"\x4c\x00\x02\x15")) or \
-            ((self.mode & ScannerMode.MODE_EDDYSTONE) and (pkt[19:21] == b"\xaa\xfe")) or \
-            ((self.mode & ScannerMode.MODE_ESTIMOTE) and (pkt[23:25] == b"\x5d\x01")) or \
-            ((self.mode & ScannerMode.MODE_CJMONITOR) and (pkt[27:29] == b"\xfe\x10")) or \
-            ((self.mode & ScannerMode.MODE_EXPOSURE_NOTIFICATION) and (pkt[19:21] == EXPOSURE_NOTIFICATION_UUID)) or \
-            ((self.mode & ScannerMode.MODE_ESTIMOTE) and (pkt[19:21] == b"\x9a\xfe"))):
+        if not self.kwtree.search(payload):
             return
 
         bt_addr = bt_addr_to_string(pkt[7:13])
         rssi = bin_to_int(pkt[-1])
         # strip bluetooth address and parse packet
-        packet = parse_packet(pkt[14:-1])
+        packet = parse_packet(payload)
 
         # return if packet was not an beacon advertisement
         if not packet:

setup.py

@@ -13,7 +13,7 @@
 setup(
     name='beacontools',
 
-    version='2.0.1',
+    version='2.0.2',
 
     description='A Python library for working with various types of Bluetooth LE Beacons.',
     long_description=long_description,
@@ -52,7 +52,8 @@
     # requirements files see:
     # https://packaging.python.org/en/latest/requirements.html
     install_requires=[
-        'construct>=2.9.52,<2.11'
+        'construct>=2.9.52,<2.11',
+        'ahocorapy==1.6.1'
     ],
 
     # List additional groups of dependencies here (e.g. development
@@ -63,10 +64,10 @@
         'scan': ['PyBluez==0.22'] if sys.platform.startswith("linux") else [],
         'dev': ['check-manifest'],
         'test': [
-            'coveralls==1.5.1',
-            'pytest==5.4.3',
-            'pytest-cov==2.10.0',
-            'mock==3.0.5',
+            'coveralls~=2.1',
+            'pytest~=6.0',
+            'pytest-cov~=2.10',
+            'mock~=4.0',
             'check-manifest==0.42',
             'pylint',
             'readme_renderer',

tests/test_scanner.py

@@ -19,7 +19,8 @@ class TestScanner(unittest.TestCase):
 
     def setUp(self):
         # mock import so that tests can run without PyBluez installed
-        sys.modules['bluetooth._bluetooth'] = MagicMock()
+        sys.modules['bluetooth'] = MagicMock()
+        sys.platform = "linux"
 
     def test_invalid_device_filters(self):
         """Test creation of device filters without arguments"""
@@ -395,6 +396,19 @@ def test_multiple_filters2(self):
         scanner._mon.process_packet(pkt)
         self.assertEqual(callback.call_count, 3)
 
+    def test_exposure_notification(self):
+        callback = MagicMock()
+        scanner = BeaconScanner(callback, packet_filter=[ExposureNotificationFrame])
+        # Android and iOS seem to use slightly different packets
+        android_pkt = b"\x04\x3E\x28\x02\x01\x03\x01\xBB\x7E\xB5\x2B\x86\x79\x1C\x03\x03\x6F\xFD\x17\x16"\
+                      b"\x6F\xFD\x2C\xFB\x0D\xE0\x2B\x33\xD2\x0C\x5C\x27\x61\x12\x38\xE2\xD1\x07\x42\xB5"\
+                      b"\x6E\xE5\xB8"
+        scanner._mon.process_packet(android_pkt)
+        ios_pkt = b"\x04\x3E\x2B\x02\x01\x03\x01\x08\xE6\xAE\x33\x0B\x3F\x1F\x02\x01\x1A\x03\x03\x6F"\
+                  b"\xFD\x17\x16\x6F\xFD\xE9\x32\xE8\xB0\x68\x8D\xFA\xEC\x00\x62\xB7\xD6\xD3\x5E\xEF"\
+                  b"\xB5\xEE\xAA\x91\xAC\xBA"
+        scanner._mon.process_packet(ios_pkt)
+        self.assertEqual(callback.call_count, 2)
 
 if __name__ == "__main__":
     unittest.main()