From 8bd2199bea0471acd46c5700045c696bd6a02e4c Mon Sep 17 00:00:00 2001 From: Raymond LeClair Date: Wed, 13 Sep 2023 17:58:38 +0000 Subject: [PATCH 1/8] Log configuration and use consistent environment and code topic variable names --- couchdb-saver/couchdb_saver_pub_sub.py | 25 ++++++++++++++++++++----- example.env | 9 +++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/couchdb-saver/couchdb_saver_pub_sub.py b/couchdb-saver/couchdb_saver_pub_sub.py index 7fb2425..a4aaf2c 100644 --- a/couchdb-saver/couchdb_saver_pub_sub.py +++ b/couchdb-saver/couchdb_saver_pub_sub.py @@ -22,7 +22,7 @@ class CouchDBSaverPubSub(BaseMQTTPubSub): Args: BaseMQTTPubSub (BaseMQTTPubSub): parent class written in the EdgeTech Core module """ - + # TODO: Include HOSTNAME? def __init__( self: Any, sensor_save_topic: str, @@ -75,6 +75,21 @@ def __init__( sleep(1) self.publish_registration("CouchDB Saver Registration") + # Log configuration parameters + logging.info( + f"""CouchDBSaverPubSub initialized with parameters: + sensor_save_topic = {sensor_save_topic} + telemetry_save_topic = {telemetry_save_topic} + audio_save_topic = {audio_save_topic} + couchdb_error_topic = {couchdb_error_topic} + couchdb_user = {couchdb_user} + couchdb_password = {couchdb_password} + couchdb_server_ip = {couchdb_server_ip} + device_ip = {device_ip} + debug = {debug} + """ + ) + def _to_save_callback( self: Any, _client: mqtt.Client, _userdata: Dict[Any, Any], msg: Any ) -> None: @@ -137,14 +152,14 @@ def main(self: Any) -> None: if __name__ == "__main__": saver = CouchDBSaverPubSub( - sensor_save_topic=str(os.environ.get("SENSOR_TOPIC")), - telemetry_save_topic=str(os.environ.get("TELEMETRY_TOPIC")), - audio_save_topic=str(os.environ.get("AUDIO_TOPIC")), + mqtt_ip=str(os.environ.get("MQTT_IP")), + sensor_save_topic=str(os.environ.get("SENSOR_SAVE_TOPIC")), + telemetry_save_topic=str(os.environ.get("TELEMETRY_SAVE_TOPIC")), + audio_save_topic=str(os.environ.get("AUDIO_SAVE_TOPIC")), couchdb_error_topic=str(os.environ.get("COUCHDB_ERROR_TOPIC")), couchdb_user=str(os.environ.get("COUCHDB_USER")), couchdb_password=str(os.environ.get("COUCHDB_PASSWORD")), couchdb_server_ip=str(os.environ.get("COUCHDB_SERVER_IP_ADDR")), device_ip=str(os.environ.get("DEVICE_IP")), - mqtt_ip=str(os.environ.get("MQTT_IP")), ) saver.main() diff --git a/example.env b/example.env index 1f53743..135b5b8 100644 --- a/example.env +++ b/example.env @@ -1,10 +1,11 @@ HOSTNAME=pinebuoy-001 -SENSOR_TOPIC=/AISonobuoy/${HOSTNAME}/AIS/edgetech-daisy/bytestring -TELEMETRY_TOPIC=/AISonobuoy/${HOSTNAME}/telemetry/telemetry/string -AUDIO_TOPIC=/AISonobuoy/${HOSTNAME}/Audio/edgetech-audio-recorder/string -COUCHDB_ERROR_TOPIC=/AISonobuoy/${HOSTNAME}/couchdbsaver/couchdb/errstring MQTT_IP=mqtt +SENSOR_SAVE_TOPIC=/AISonobuoy/${HOSTNAME}/AIS/edgetech-daisy/bytestring +TELEMETRY_SAVE_TOPIC=/AISonobuoy/${HOSTNAME}/telemetry/telemetry/string +AUDIO_SAVE_TOPIC=/AISonobuoy/${HOSTNAME}/Audio/edgetech-audio-recorder/string +COUCHDB_ERROR_TOPIC=/AISonobuoy/${HOSTNAME}/couchdbsaver/couchdb/errstring COUCHDB_USER= COUCHDB_PASSWORD= COUCHDB_SERVER_IP_ADDR= DEVICE_IP= +LOG_LEVEL=DEBUG From 27ef4bf4a8c2e9f1cedc23c11da2ae6b21cba1cb Mon Sep 17 00:00:00 2001 From: Raymond LeClair Date: Wed, 13 Sep 2023 18:06:12 +0000 Subject: [PATCH 2/8] Import logging --- couchdb-saver/couchdb_saver_pub_sub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchdb-saver/couchdb_saver_pub_sub.py b/couchdb-saver/couchdb_saver_pub_sub.py index a4aaf2c..fa1a3d7 100644 --- a/couchdb-saver/couchdb_saver_pub_sub.py +++ b/couchdb-saver/couchdb_saver_pub_sub.py @@ -4,6 +4,7 @@ """ import os import json +import logging from time import sleep from typing import Any, Dict From 3abe9c6ac41d5d29bd7ee46f6c2529439733b8e4 Mon Sep 17 00:00:00 2001 From: Raymond LeClair Date: Fri, 6 Oct 2023 13:38:30 +0000 Subject: [PATCH 3/8] Add log level --- couchdb-saver/couchdb_saver_pub_sub.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/couchdb-saver/couchdb_saver_pub_sub.py b/couchdb-saver/couchdb_saver_pub_sub.py index fa1a3d7..c7e28e9 100644 --- a/couchdb-saver/couchdb_saver_pub_sub.py +++ b/couchdb-saver/couchdb_saver_pub_sub.py @@ -35,6 +35,7 @@ def __init__( couchdb_server_ip: str, device_ip: str, debug: bool = False, + log_level: str = "INFO", **kwargs: Any, ) -> None: """The CouchDBSaverPubSub takes MQTT topics to write data from and CouchDB authentication @@ -51,6 +52,8 @@ def __init__( couchdb_user (str): the username for CouchDB authentication couchdb_password (str): the password for CouchDB authentication couchdb_server_ip (str): the IP address for couchDB authentication + log_level (str): One of 'NOTSET', 'DEBUG', 'INFO', 'WARN', + 'WARNING', 'ERROR', 'FATAL', 'CRITICAL' device_ip (str): the IP of the current device """ # to override any of the BaseMQTTPubSub attributes @@ -66,6 +69,7 @@ def __init__( self.couchdb_server_ip = couchdb_server_ip self.device_ip = device_ip self.debug = debug + self.log_level = log_level # open schema file for validation with open("couchdb_saver.schema", "r", encoding="utf-8") as file_pointer: @@ -88,6 +92,7 @@ def __init__( couchdb_server_ip = {couchdb_server_ip} device_ip = {device_ip} debug = {debug} + log_level = {log_level} """ ) From 3649c368f9e768a71131ae18fbb4794fd1d68a57 Mon Sep 17 00:00:00 2001 From: Raymond LeClair Date: Fri, 6 Oct 2023 13:40:24 +0000 Subject: [PATCH 4/8] Allow None type to produce meaningful error messges --- couchdb-saver/couchdb_saver_pub_sub.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/couchdb-saver/couchdb_saver_pub_sub.py b/couchdb-saver/couchdb_saver_pub_sub.py index c7e28e9..3ae7fde 100644 --- a/couchdb-saver/couchdb_saver_pub_sub.py +++ b/couchdb-saver/couchdb_saver_pub_sub.py @@ -158,14 +158,15 @@ def main(self: Any) -> None: if __name__ == "__main__": saver = CouchDBSaverPubSub( - mqtt_ip=str(os.environ.get("MQTT_IP")), - sensor_save_topic=str(os.environ.get("SENSOR_SAVE_TOPIC")), - telemetry_save_topic=str(os.environ.get("TELEMETRY_SAVE_TOPIC")), - audio_save_topic=str(os.environ.get("AUDIO_SAVE_TOPIC")), - couchdb_error_topic=str(os.environ.get("COUCHDB_ERROR_TOPIC")), - couchdb_user=str(os.environ.get("COUCHDB_USER")), - couchdb_password=str(os.environ.get("COUCHDB_PASSWORD")), - couchdb_server_ip=str(os.environ.get("COUCHDB_SERVER_IP_ADDR")), - device_ip=str(os.environ.get("DEVICE_IP")), + mqtt_ip=os.environ.get("MQTT_IP"), + sensor_save_topic=os.environ.get("SENSOR_SAVE_TOPIC"), + telemetry_save_topic=os.environ.get("TELEMETRY_SAVE_TOPIC"), + audio_save_topic=os.environ.get("AUDIO_SAVE_TOPIC"), + couchdb_error_topic=os.environ.get("COUCHDB_ERROR_TOPIC"), + couchdb_user=os.environ.get("COUCHDB_USER"), + couchdb_password=os.environ.get("COUCHDB_PASSWORD"), + couchdb_server_ip=os.environ.get("COUCHDB_SERVER_IP_ADDR"), + device_ip=os.environ.get("DEVICE_IP"), + log_level=os.environ.get("LOG_LEVEL") ) saver.main() From 6f4b0baed7f4db824fd2685c3ce4dd639c8a2b66 Mon Sep 17 00:00:00 2001 From: Raymond LeClair Date: Fri, 6 Oct 2023 13:41:17 +0000 Subject: [PATCH 5/8] Use named environment file --- couchdb-saver.env | 11 +++++++++++ example.env | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 couchdb-saver.env delete mode 100644 example.env diff --git a/couchdb-saver.env b/couchdb-saver.env new file mode 100644 index 0000000..53e7289 --- /dev/null +++ b/couchdb-saver.env @@ -0,0 +1,11 @@ +HOSTNAME= +MQTT_IP= +SENSOR_SAVE_TOPIC=/Multimodal/$HOSTNAME/Sensor_Save/edgetech-audio-recorder/string +TELEMETRY_TOPIC=/Multimodal/$HOSTNAME/Telemetry/edgetech-audio-recorder/string +AUDIO_SAVE_TOPIC=/Multimodal/$HOSTNAME/Audio_Save/edgetech-audio-recorder/string +COUCHDB_ERROR_TOPIC=/Multimodal/$HOSTNAME/CouchDB_Error/edgetech-audio-recorder/string +COUCHDB_USER= +COUCHDB_PASSWORD= +COUCHDB_SERVER_IP_ADDR= +DEVICE_IP= +LOG_LEVEL=INFO diff --git a/example.env b/example.env deleted file mode 100644 index 135b5b8..0000000 --- a/example.env +++ /dev/null @@ -1,11 +0,0 @@ -HOSTNAME=pinebuoy-001 -MQTT_IP=mqtt -SENSOR_SAVE_TOPIC=/AISonobuoy/${HOSTNAME}/AIS/edgetech-daisy/bytestring -TELEMETRY_SAVE_TOPIC=/AISonobuoy/${HOSTNAME}/telemetry/telemetry/string -AUDIO_SAVE_TOPIC=/AISonobuoy/${HOSTNAME}/Audio/edgetech-audio-recorder/string -COUCHDB_ERROR_TOPIC=/AISonobuoy/${HOSTNAME}/couchdbsaver/couchdb/errstring -COUCHDB_USER= -COUCHDB_PASSWORD= -COUCHDB_SERVER_IP_ADDR= -DEVICE_IP= -LOG_LEVEL=DEBUG From 1beb19085fb8af4d638e1d8dc5139e9b4e73a4c4 Mon Sep 17 00:00:00 2001 From: Raymond LeClair Date: Mon, 9 Oct 2023 18:45:21 +0000 Subject: [PATCH 6/8] Use standard docker-compose layout --- docker-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e0559e..c27cafa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,15 +16,15 @@ services: build: context: ./couchdb-saver dockerfile: ./Dockerfile + restart: unless-stopped volumes: - /home/mobian/sensor-data:/sensor-data - restart: unless-stopped - depends_on: - - mqtt - env_file: - - .env logging: driver: "json-file" options: max-size: "10M" - max-file: "10" \ No newline at end of file + max-file: "10" + depends_on: + - mqtt + env_file: + - .env From dd99eb39a1733e858700c5a3c8b31b3e12268010 Mon Sep 17 00:00:00 2001 From: Raymond LeClair Date: Mon, 9 Oct 2023 18:48:46 +0000 Subject: [PATCH 7/8] Resolve code quality issues --- couchdb-saver/couchdb_saver_pub_sub.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/couchdb-saver/couchdb_saver_pub_sub.py b/couchdb-saver/couchdb_saver_pub_sub.py index 3ae7fde..896d7b6 100644 --- a/couchdb-saver/couchdb_saver_pub_sub.py +++ b/couchdb-saver/couchdb_saver_pub_sub.py @@ -23,6 +23,7 @@ class CouchDBSaverPubSub(BaseMQTTPubSub): Args: BaseMQTTPubSub (BaseMQTTPubSub): parent class written in the EdgeTech Core module """ + # TODO: Include HOSTNAME? def __init__( self: Any, @@ -158,15 +159,15 @@ def main(self: Any) -> None: if __name__ == "__main__": saver = CouchDBSaverPubSub( - mqtt_ip=os.environ.get("MQTT_IP"), - sensor_save_topic=os.environ.get("SENSOR_SAVE_TOPIC"), - telemetry_save_topic=os.environ.get("TELEMETRY_SAVE_TOPIC"), - audio_save_topic=os.environ.get("AUDIO_SAVE_TOPIC"), - couchdb_error_topic=os.environ.get("COUCHDB_ERROR_TOPIC"), - couchdb_user=os.environ.get("COUCHDB_USER"), - couchdb_password=os.environ.get("COUCHDB_PASSWORD"), - couchdb_server_ip=os.environ.get("COUCHDB_SERVER_IP_ADDR"), - device_ip=os.environ.get("DEVICE_IP"), - log_level=os.environ.get("LOG_LEVEL") + mqtt_ip=str(os.environ.get("MQTT_IP")), + sensor_save_topic=str(os.environ.get("SENSOR_SAVE_TOPIC")), + telemetry_save_topic=str(os.environ.get("TELEMETRY_SAVE_TOPIC")), + audio_save_topic=str(os.environ.get("AUDIO_SAVE_TOPIC")), + couchdb_error_topic=str(os.environ.get("COUCHDB_ERROR_TOPIC")), + couchdb_user=str(os.environ.get("COUCHDB_USER")), + couchdb_password=str(os.environ.get("COUCHDB_PASSWORD")), + couchdb_server_ip=str(os.environ.get("COUCHDB_SERVER_IP_ADDR")), + device_ip=str(os.environ.get("DEVICE_IP")), + log_level=str(os.environ.get("LOG_LEVEL")), ) saver.main() From f3c22b337be8d84bf0df3de5266244b6155fa530 Mon Sep 17 00:00:00 2001 From: Luke Berndt Date: Thu, 22 Feb 2024 13:58:31 -0500 Subject: [PATCH 8/8] ENV File cleanup --- couchdb-saver.env | 11 ++++++----- docker-compose.yml | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/couchdb-saver.env b/couchdb-saver.env index 53e7289..97cf148 100644 --- a/couchdb-saver.env +++ b/couchdb-saver.env @@ -1,9 +1,10 @@ +PROJECT_NAME= HOSTNAME= -MQTT_IP= -SENSOR_SAVE_TOPIC=/Multimodal/$HOSTNAME/Sensor_Save/edgetech-audio-recorder/string -TELEMETRY_TOPIC=/Multimodal/$HOSTNAME/Telemetry/edgetech-audio-recorder/string -AUDIO_SAVE_TOPIC=/Multimodal/$HOSTNAME/Audio_Save/edgetech-audio-recorder/string -COUCHDB_ERROR_TOPIC=/Multimodal/$HOSTNAME/CouchDB_Error/edgetech-audio-recorder/string +MQTT_IP=mqtt +SENSOR_SAVE_TOPIC=/${PROJECT_NAME}/${HOSTNAME}/Sensor_Save/edgetech-audio-recorder/string +TELEMETRY_TOPIC=/${PROJECT_NAME}/${HOSTNAME}/Telemetry/edgetech-audio-recorder/string +AUDIO_SAVE_TOPIC=/${PROJECT_NAME}/${HOSTNAME}/Audio_Save/edgetech-audio-recorder/string +COUCHDB_ERROR_TOPIC=/${PROJECT_NAME}/${HOSTNAME}/CouchDB_Error/edgetech-audio-recorder/string COUCHDB_USER= COUCHDB_PASSWORD= COUCHDB_SERVER_IP_ADDR= diff --git a/docker-compose.yml b/docker-compose.yml index c27cafa..0e6575a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,4 +27,5 @@ services: depends_on: - mqtt env_file: - - .env + - path: ./couchdb-saver.env + required: false