Fixed conflict with hydra logger

facebookresearch · Aug 8, 2023 · b4278a5 · b4278a5
1 parent ec77be2
commit b4278a5
Showing 1 changed file with 40 additions and 16 deletions.
diff --git a/mephisto/configs/logging.py b/mephisto/configs/logging.py
@@ -9,17 +9,41 @@
 from datetime import datetime
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-CONSOLE_LOG_LEVEL = os.environ.get("CONSOLE_LOG_LEVEL", "INFO")
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
+WRITE_LOG_TO_FILE = os.environ.get("WRITE_LOG_TO_FILE", '0')
 
 _now = datetime.now()
 date_string = _now.strftime("%Y-%m-%d")
 time_string = _now.strftime("%H-%M-%S")
-executed_filename = os.path.splitext(os.path.basename(sys.argv[0]))[0]
 
-# Create dirs recursivelly if they do not exist
-os.makedirs(os.path.join(BASE_DIR, "outputs", date_string, time_string), exist_ok=True)
 
-# Logging
+def get_log_handlers():
+    """ We enable module-level loggers via env variable (that we can set in the console),
+        so that hydra doesn't create an empty file for every module-level logger
+    """
+    handlers = ["console"]
+    if WRITE_LOG_TO_FILE == '1':
+        handlers.append("file")
+        # Create dirs recursivelly if they do not exist
+        os.makedirs(
+            os.path.join(BASE_DIR, "outputs", date_string, time_string),
+            exist_ok=True
+        )
+    return handlers
+
+
+def get_log_filename():
+    """ Compose logfile path formatted same way as hydra """
+    executed_filename = os.path.splitext(os.path.basename(sys.argv[0]))[0]
+    return os.path.join(
+        BASE_DIR, "outputs", date_string, time_string, f'{executed_filename}.log',
+    )
+
+
+log_handlers = get_log_handlers()
+log_filename = get_log_filename()
+
+# Logging config
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": True,
@@ -30,24 +54,24 @@
     },
     "handlers": {
         "console": {
-            "level": CONSOLE_LOG_LEVEL,
+            "level": LOG_LEVEL,
             "class": "logging.StreamHandler",
             "formatter": "default",
         },
-        "file": {
-            "level": "DEBUG",
-            "class": "logging.FileHandler",
-            "filename": os.path.join(
-                BASE_DIR, "outputs", date_string, time_string, f'{executed_filename}.log',
-            ),
-            "formatter": "default",
-        },
+        **({
+            "file": {
+                "level": LOG_LEVEL,
+                "class": "logging.FileHandler",
+                "filename": log_filename,
+                "formatter": "default",
+            }
+        } if "file" in log_handlers else {}),
     },
     "loggers": {
         "": {
-            "handlers": ["console", "file"],
+            "handlers": log_handlers,
             "propagate": True,
-            "level": CONSOLE_LOG_LEVEL,
+            "level": LOG_LEVEL,
         },
     },
 }