update

Signed-off-by: Praneeth Bedapudi <[email protected]>

smartlog/setup.py

@@ -18,7 +18,7 @@
 EMAIL = "[email protected]"
 AUTHOR = "BEDAPUDI PRANEETH"
 REQUIRES_PYTHON = ">=3.6.0"
-VERSION = "0.0.1.dev23"
+VERSION = "0.0.1.dev24"
 
 # What packages are required for this module to be executed?
 REQUIRED = ["requests", "liteindex"]

smartlog/smartlogger/smartlogger.py

@@ -33,48 +33,58 @@ def upload_data(db_file, batch_size=512):
         name = os.path.splitext(os.path.basename(db_file))[0]
 
         try:
-            logs_index = DefinedIndex(
-                "logs",
-                db_path=os.path.join(log_dir, db_file)
-            )
+            logs_index = DefinedIndex("logs", db_path=os.path.join(log_dir, db_file))
 
             key_value_index = DefinedIndex(
-                "key_value",
-                db_path=os.path.join(log_dir, db_file)
+                "key_value", db_path=os.path.join(log_dir, db_file)
             )
         except:
             return
 
         total_n_logs_popped = 0
         total_n_key_value_popped = 0
 
+        logs_index_total_len = logs_index.count()
+        key_value_index_total_len = key_value_index.count()
+
+        if logs_index_total_len or key_value_index_total_len:
+            print(
+                f"smartlogger {name}: syncing {logs_index_total_len} logs, {key_value_index_total_len} key values"
+            )
+
+        error_already_printed = False
+
         while True:
             logs_index_popped_data = logs_index.pop(n=batch_size)
 
             n_logs_popped = None
             n_key_value_popped = None
-            
+
             if logs_index_popped_data:
                 n_logs_popped = len(logs_index_popped_data)
 
                 for k in logs_index_popped_data:
                     logs_index_popped_data[k]["app_name"] = name
 
-
                 try:
                     resp = requests.post(
-                        f"{url}/logs", data=pickle.dumps(logs_index_popped_data, protocol=pickle.HIGHEST_PROTOCOL)
+                        f"{url}/logs",
+                        data=pickle.dumps(
+                            logs_index_popped_data, protocol=pickle.HIGHEST_PROTOCOL
+                        ),
                     ).json()
 
-
                     logs_index_popped_data = None
 
                     if not resp["success"] == True:
-                        1/0
+                        1 / 0
 
+                    error_already_printed = False
                 except Exception as ex:
-                    pass
-
+                    if not error_already_printed:
+                        print(f"smartlogger {name}: error syncing logs: {ex}")
+                        error_already_printed = True
+
             key_value_index_popped_data = key_value_index.pop(n=batch_size)
 
             if key_value_index_popped_data:
@@ -83,17 +93,19 @@ def upload_data(db_file, batch_size=512):
                 for k in key_value_index_popped_data:
                     key_value_index_popped_data[k]["app_name"] = name
 
-
                 try:
                     resp = requests.post(
-                        f"{url}/key_values", data=pickle.dumps(key_value_index_popped_data, protocol=pickle.HIGHEST_PROTOCOL)
+                        f"{url}/key_values",
+                        data=pickle.dumps(
+                            key_value_index_popped_data,
+                            protocol=pickle.HIGHEST_PROTOCOL,
+                        ),
                     ).json()
 
-
                     key_value_index_popped_data = None
 
                     if not resp["success"] == True:
-                        1/0
+                        1 / 0
 
                 except Exception as ex:
                     pass
@@ -103,23 +115,25 @@ def upload_data(db_file, batch_size=512):
 
             if not n_logs_popped and not n_key_value_popped:
                 if total_n_logs_popped > 0 or total_n_key_value_popped > 0:
-                    print(f"smartlogger {name}: synced {total_n_logs_popped} logs, {total_n_key_value_popped} key values")
+                    print(
+                        f"smartlogger {name}: synced {total_n_logs_popped} logs, {total_n_key_value_popped} key values"
+                    )
+                    logs_index.vaccum()
+                    key_value_index.vaccum()
                 return
 
-
-
     while True:
         for db_file in glob(os.path.join(log_dir, "*.db")):
             upload_data(db_file, batch_size=batch_size)
-            
+
         time.sleep(int(os.getenv("SYNC_SLEEP", 10)))
 
 
 class SmartLogger:
     def __init__(self, name, dir="./", log_to_console=False):
         self.name = name
         self.log_to_console = log_to_console
-        
+
         os.makedirs(dir, exist_ok=True)
         db_path = os.path.join(dir, f"{self.name}.db")
 
@@ -153,14 +167,16 @@ def __init__(self, name, dir="./", log_to_console=False):
         )
 
     def _log(self, id, level, *messages, stage=None, tags=[]):
+        timestamp = time.time()
+
         self.logs_index.update(
             {
                 str(uuid.uuid4()): {
                     "u_id": str(id),
                     "stage": stage,
                     "level": level,
                     "messages": [str(m) for m in messages],
-                    "time": time.time(),
+                    "time": timestamp,
                     "tags": tags,
                 }
             }