Issue 17 sqlite

.gitignore

@@ -129,3 +129,4 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+/test_database.db

main.py

@@ -16,17 +16,16 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 from __future__ import annotations
+
 import argparse
+import asyncio
+import time
 from typing import Optional
-from typing import Callable
-import serial
 
 import common
-from digi.xbee import devices as xbee_devices
-from digi.xbee.models import message as xbee_message
-import asyncio
 import protocol
 import protocol_factory
+import serverdatabase
 
 _SW_VERSION = 10000
 
@@ -72,9 +71,40 @@ def __init__(self, ip: str, port: int, com: str, baud: int, xbee_mac: str, verbo
         self._event_loop = None
         self._on_stop = None
 
+        self._database = serverdatabase.ServerDatabase("staging")
+        self._initialise_database()
+
         self._car_clients = {}
         self._gui_clients = {}
 
+    def _initialise_database(self) -> None:
+        """
+        Initialise the staging database.
+        """
+        sensors = ["rpm", "water_temp_c", "tps_perc", "battery_mv", "ext_5v_mv",
+                   "fuel_flow", "lambda", "speed_kph", "evo_scan_1", "evo_scan_2",
+                   "evo_scan_3", "evo_scan_4", "evo_scan_5", "evo_scan_6", "evo_scan_7",
+                   "status_ecu_connected", "status_engine", "status_battery", "status_logging",
+                   "inj_time", "inj_duty_cycle", "lambda_pid_adj", "lambda_pid_target",
+                   "advance", "ride_height_fl_cm", "ride_height_fr_cm", "ride_height_flw_cm",
+                   "ride_height_rear_cm", "lap_time_s", "accel_fl_x_mg", "accel_fl_y_mg",
+                   "accel_fl_z_mg"]
+
+        for sensor in sensors:
+            self._database.create_sensor_table(sensor, ["value"])
+
+    def _save_sensor_data_to_database(self, sensor_data: list) -> None:
+        """
+        Save a list of sensor data to the staging database.
+        :param sensor_data: The list of sensor data. (name, time, values).
+        """
+        # Insert the data into database.
+        for sensor in sensor_data:
+            name, time_ms, value = sensor
+            self._database.insert_sensor_data(name, time_ms, (value,))
+
+        self._database.commit()
+
     def __enter__(self) -> Server:
         """
         Enter for use with "with as"
@@ -127,6 +157,16 @@ def _on_acpdu(self, factory: protocol_factory.ProtocolFactoryBase, header: proto
         :param factory: The factory that received the frame.
         """
         self._logger.info(f"Handling ACPDU from factory {factory.__hash__()}")
+
+        time_ms = int(time.time() * 1000)
+        sensor_data = [("rpm", time_ms, rpm), ("water_temp_c", time_ms, water_temp_c),
+                       ("tps_perc", time_ms, tps_perc), ("battery_mv", time_ms, battery_mv),
+                       ("ext_5v_mv", time_ms, external_5v_mv), ("fuel_flow", time_ms, fuel_flow),
+                       ("lambda", time_ms, lambda_value), ("speed_kph", time_ms, speed_kph)]
+
+        # Insert the data into database.
+        self._save_sensor_data_to_database(sensor_data)
+
         for _, client in self._gui_clients.items():
             self._logger.info(
                 f"Routing ACPDU to client {client.client_name} through factory {client.factory.__hash__()}")
@@ -141,6 +181,16 @@ def _on_aapdu(self, factory: protocol_factory.ProtocolFactoryBase, header: proto
         :param factory: The factory that received the frame.
         """
         self._logger.info(f"Handling AAPDU from factory {factory.__hash__()}")
+
+        time_ms = int(time.time() * 1000)
+        sensor_data = [("evo_scan_1", time_ms, evo_scanner1), ("evo_scan_2", time_ms, evo_scanner2),
+                       ("evo_scan_3", time_ms, evo_scanner3), ("evo_scan_4", time_ms, evo_scanner4),
+                       ("evo_scan_5", time_ms, evo_scanner5), ("evo_scan_6", time_ms, evo_scanner6),
+                       ("evo_scan_7", time_ms, evo_scanner7)]
+
+        # Insert the data into database.
+        self._save_sensor_data_to_database(sensor_data)
+
         for _, client in self._gui_clients.items():
             self._logger.info(
                 f"Routing AAPDU to client {client.client_name} through factory {client.factory.__hash__()}")
@@ -154,6 +204,16 @@ def _on_adpdu(self, factory: protocol_factory.ProtocolFactoryBase, header: proto
         :param factory: The factory that received the frame.
         """
         self._logger.info(f"Handling ADPDU from factory {factory.__hash__()}")
+
+        time_ms = int(time.time() * 1000)
+        sensor_data = [("status_ecu_connected", time_ms, ecu_status),
+                       ("status_engine", time_ms, engine_status),
+                       ("status_battery", time_ms, battery_status),
+                       ("status_logging", time_ms, car_logging_status)]
+
+        # Insert the data into database.
+        self._save_sensor_data_to_database(sensor_data)
+
         for _, client in self._gui_clients.items():
             self._logger.info(
                 f"Routing ADPDU to client {client.client_name} through factory {client.factory.__hash__()}")
@@ -167,6 +227,17 @@ def _on_appdu(self, factory: protocol_factory.ProtocolFactoryBase, header: proto
         :param factory: The factory that received the frame.
         """
         self._logger.info(f"Handling APPDU from factory {factory.__hash__()}")
+
+        time_ms = int(time.time() * 1000)
+        sensor_data = [("inj_time", time_ms, injection_time),
+                       ("inj_duty_cycle", time_ms, injection_duty_cycle),
+                       ("lambda_pid_adj", time_ms, lambda_pid_adjust),
+                       ("lambda_pid_target", time_ms, lambda_pid_target),
+                       ("advance", time_ms, advance)]
+
+        # Insert the data into database.
+        self._save_sensor_data_to_database(sensor_data)
+
         for _, client in self._gui_clients.items():
             self._logger.info(
                 f"Routing APPDU to client {client.client_name} through factory {client.factory.__hash__()}")
@@ -181,6 +252,16 @@ def _on_aspdu(self, factory: protocol_factory.ProtocolFactoryBase, header: proto
         :param factory: The factory that received the frame.
         """
         self._logger.info(f"Handling ASPDU from factory {factory.__hash__()}")
+
+        time_ms = int(time.time() * 1000)
+        sensor_data = [("ride_height_fl_cm", time_ms, ride_height_fl_cm),
+                       ("ride_height_fr_cm", time_ms, ride_height_fr_cm),
+                       ("ride_height_flw_cm", time_ms, ride_height_flw_cm),
+                       ("ride_height_rear_cm", time_ms, ride_height_rear_cm)]
+
+        # Insert the data into database.
+        self._save_sensor_data_to_database(sensor_data)
+
         for _, client in self._gui_clients.items():
             self._logger.info(
                 f"Routing ASPDU to client {client.client_name} through factory {client.factory.__hash__()}")
@@ -194,6 +275,16 @@ def _on_ampdu(self, factory: protocol_factory.ProtocolFactoryBase, header: proto
         :param factory: The factory that received the frame.
         """
         self._logger.info(f"Handling AMPDU from factory {factory.__hash__()}")
+
+        time_ms = int(time.time() * 1000)
+        sensor_data = [("lap_time_s", time_ms, lap_timer_s),
+                       ("accel_fl_x_mg", time_ms, accel_fl_x_mg),
+                       ("accel_fl_y_mg", time_ms, accel_fl_y_mg),
+                       ("accel_fl_z_mg", time_ms, accel_fl_z_mg)]
+
+        # Insert the data into database.
+        self._save_sensor_data_to_database(sensor_data)
+
         for _, client in self._gui_clients.items():
             self._logger.info(
                 f"Routing AMPDU to client {client.client_name} through factory {client.factory.__hash__()}")

serverdatabase.py

@@ -0,0 +1,80 @@
+"""
+    Southampton University Formula Student Team Intermediate Server
+    Copyright (C) 2021 Nathan Rowley-Smith
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import sqlite3
+
+
+def _get_comma_separated_entry(entries: tuple or list) -> str:
+    entries_str = ""
+    for entry in entries:
+        entries_str = f"{entries_str}{entry},"
+
+    # Remove the trailing ","
+    return entries_str[:-1]
+
+
+class ServerDatabase:
+    def __init__(self, name: str):
+        self._con = sqlite3.connect(f"{name}.db")
+
+    def create_sensor_table(self, sensor: str, columns: list) -> None:
+        """
+        Create the sensor tables in the server database.
+        :param sensor: The sensor name.
+        :param columns: The column names for each sensor (minus the time column).
+        :return: None
+        """
+        cur = self._con.cursor()
+
+        # Create the table if they do not exist already.
+        cur.execute(f"CREATE TABLE IF NOT EXISTS {sensor} (time,{_get_comma_separated_entry(columns)})")
+
+    def insert_sensor_data(self, sensor: str, time: int, data: tuple) -> None:
+        """
+        Insert sensor data in the database.
+        :param time: The timestamp of the data in the format of Epoch.
+        :param sensor: The sensor name to insert into.
+        :param data: The sensor data to insert.
+        :return: None
+        """
+        cur = self._con.cursor()
+
+        cur.execute(f"INSERT INTO {sensor} VALUES ({time},{_get_comma_separated_entry(data)})")
+
+        self._con.commit()
+
+    def select_sensor_data_between_times(self, sensor: str, times: list) -> list:
+        """
+        Get sensor data between two times points from the sensor table (time column included)
+        :param sensor: The sensor to select from.
+        :param times: The times [low, high] to get data between.
+        :return A list of sensor data tuples for between the times.
+        """
+        data = []
+        cur = self._con.cursor()
+
+        for row in cur.execute(f"SELECT * FROM {sensor} WHERE time BETWEEN {times[0]} AND {times[1]}"):
+            data.append(row)
+
+        return data
+
+    def commit(self):
+        """
+        Commit the database to file.
+        """
+        self._con.commit()

test_serverdatabase.py

@@ -0,0 +1,58 @@
+import os
+import time
+import unittest
+
+import serverdatabase
+
+
+class TestServerDatabase(unittest.TestCase):
+    def test_initialise(self):
+        serverdatabase.ServerDatabase("test_database")
+        self.assertTrue(os.path.isfile("test_database.db"))
+
+    def test_sensors_table(self):
+        test_vectors = [("sense0", 1234), ("sense1", 4321)]
+
+        database = serverdatabase.ServerDatabase("test_database")
+
+        for vec in test_vectors:
+            sensor, value = vec
+
+            database.create_sensor_table(sensor, ["value"])
+
+            time_s = int(time.time())
+            database.insert_sensor_data(sensor, time_s, (value,))
+
+            sensor_data = database.select_sensor_data_between_times(sensor, [time_s - 5, time_s])
+            sensor_time, sensor_val = sensor_data[0]
+            self.assertEqual(sensor_val, value)
+
+    def test_commit(self):
+        test_vectors = [("sense0", 1234), ("sense1", 4321)]
+
+        database = serverdatabase.ServerDatabase("test_database")
+
+        time_s = int(time.time())
+
+        for vec in test_vectors:
+            sensor, value = vec
+
+            database.create_sensor_table(sensor, ["value"])
+            database.insert_sensor_data(sensor, time_s, (value,))
+
+        database.commit()
+
+        del database
+
+        database = serverdatabase.ServerDatabase("test_database")
+
+        for vec in test_vectors:
+            sensor, value = vec
+
+            sensor_data = database.select_sensor_data_between_times(sensor, [time_s - 5, time_s])
+            sensor_time, sensor_val = sensor_data[0]
+            self.assertEqual(sensor_val, value)
+
+
+if __name__ == '__main__':
+    unittest.main()