From 86f1fbc9248beb4f127e611b04c46f1e33a5d848 Mon Sep 17 00:00:00 2001 From: Praneeth Bedapudi Date: Mon, 4 Sep 2023 13:50:08 +0530 Subject: [PATCH] update metrics Signed-off-by: Praneeth Bedapudi --- smartlog/setup.py | 2 +- smartlog/smartlogger/smartlogger.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/smartlog/setup.py b/smartlog/setup.py index f055d11..f9ebebd 100644 --- a/smartlog/setup.py +++ b/smartlog/setup.py @@ -18,7 +18,7 @@ EMAIL = "praneeth@bpraneeth.com" AUTHOR = "BEDAPUDI PRANEETH" REQUIRES_PYTHON = ">=3.6.0" -VERSION = "0.0.1.dev19" +VERSION = "0.0.1.dev19.1" # What packages are required for this module to be executed? REQUIRED = ["requests", "liteindex"] diff --git a/smartlog/smartlogger/smartlogger.py b/smartlog/smartlogger/smartlogger.py index 3493492..b9b02be 100644 --- a/smartlog/smartlogger/smartlogger.py +++ b/smartlog/smartlogger/smartlogger.py @@ -19,6 +19,8 @@ def upload_to_smartdash(): "--server_url", type=str, help="Smartdash server URL", required=True ) args = parser.parse_args() + + print(f"Starting sync to {args.server_url}") _upload_to_smartdash(args.save_dir, args.server_url) @@ -27,6 +29,7 @@ def _upload_to_smartdash(log_dir, url, batch_size=100): from glob import glob def upload_data(name, index_type, db_prefix): + last_sync_failed = False try: index = DefinedIndex( f"{name}_{index_type}", @@ -48,17 +51,24 @@ def upload_data(name, index_type, db_prefix): if batch: try: - resp = requests.post(f"{url}/{index_type}", json=batch).json() + resp = requests.post( + f"{url}/{index_type}", json=batch, timeout=5 + ).json() if not resp["success"] == True: 1 / 0 index.delete(keys) backoff_time = 1 + last_sync_failed = False except: if len(keys) >= 50000: index.delete(keys[:50000]) keys = keys[50000:] + if not last_sync_failed: + print(f"Failed to sync {name} logs to {url}") + last_sync_failed = True + backoff_time = min(backoff_time * 2, 60) time.sleep(backoff_time) else: