imp: logging

xoe-labs · Dec 30, 2019 · 58680b5 · 58680b5
1 parent 37f2b2a
commit 58680b5
Show file tree

Hide file tree

Showing 12 changed files with 133 additions and 87 deletions.
diff --git a/.isort.cfg b/.isort.cfg
@@ -6,4 +6,4 @@ force_grid_wrap=0
 combine_as_imports=True
 use_parentheses=True
 line_length=88
-known_third_party = click,click_pathlib,click_plugins,colorlog,dodoo_backup,dodoo_copy,dodoo_init,dodoo_run,dodoo_shell,mashumaro,pkg_resources,psycopg2,pygit2,pytest,pytest_postgresql,secure_cookie,starlette,starlette_prometheus,strawberry,uvicorn,werkzeug
+known_third_party = click,click_pathlib,click_plugins,dodoo_backup,dodoo_copy,dodoo_init,dodoo_run,dodoo_shell,mashumaro,pkg_resources,psycopg2,pygit2,pytest,pytest_postgresql,secure_cookie,starlette,starlette_prometheus,strawberry,uvicorn,werkzeug
diff --git a/dodoo-backup/tests/conftest.py b/dodoo-backup/tests/conftest.py
@@ -115,7 +115,7 @@ def main_loaded(confd, project_version_file, framework_dir, mocker):
 
     orig_framework = dodoo._framework
     mocker.patch("dodoo.create_custom_schema_layout")
-    main(framework_dir, confd, False, RUNMODE.Production, 0, project_version_file)
+    main(framework_dir, confd, False, RUNMODE.Production, 0, None, project_version_file)
     yield
     dodoo._framework = orig_framework
 

diff --git a/dodoo-copy/tests/conftest.py b/dodoo-copy/tests/conftest.py
@@ -115,7 +115,7 @@ def main_loaded(confd, project_version_file, framework_dir, mocker):
 
     orig_framework = dodoo._framework
     mocker.patch("dodoo.create_custom_schema_layout")
-    main(framework_dir, confd, False, RUNMODE.Production, 0, project_version_file)
+    main(framework_dir, confd, False, RUNMODE.Production, 0, None, project_version_file)
     yield
     dodoo._framework = orig_framework
 

diff --git a/dodoo-init/tests/conftest.py b/dodoo-init/tests/conftest.py
@@ -115,7 +115,7 @@ def main_loaded(confd, project_version_file, framework_dir, mocker):
 
     orig_framework = dodoo._framework
     mocker.patch("dodoo.create_custom_schema_layout")
-    main(framework_dir, confd, False, RUNMODE.Production, 0, project_version_file)
+    main(framework_dir, confd, False, RUNMODE.Production, 0, None, project_version_file)
     yield
     dodoo._framework = orig_framework
 

diff --git a/dodoo-run/dodoo_run/servers/__init__.py b/dodoo-run/dodoo_run/servers/__init__.py
@@ -18,7 +18,7 @@
 def server(app: ASGIApp, host: str, port: int, prod: bool = True) -> "server":
     global server
 
-    kwargs = dict(host=host, port=port, log_level="debug", debug=True)
+    kwargs = dict(host=host, port=port)
 
     if not prod:
         odooconfig = dodoo.framework().dodoo_config.Odoo

diff --git a/dodoo-run/tests/conftest.py b/dodoo-run/tests/conftest.py
@@ -150,7 +150,7 @@ def main_loaded(confd, project_version_file, framework_dir, mocker):
 
     orig_framework = dodoo._framework
     mocker.patch("dodoo.create_custom_schema_layout")
-    main(framework_dir, confd, False, RUNMODE.Develop, 0, project_version_file)
+    main(framework_dir, confd, False, RUNMODE.Develop, 0, None, project_version_file)
     yield
     dodoo._framework = orig_framework
 

diff --git a/dodoo-shell/tests/conftest.py b/dodoo-shell/tests/conftest.py
@@ -115,7 +115,7 @@ def main_loaded(confd, project_version_file, framework_dir, mocker):
 
     orig_framework = dodoo._framework
     mocker.patch("dodoo.create_custom_schema_layout")
-    main(framework_dir, confd, False, RUNMODE.Production, 0, project_version_file)
+    main(framework_dir, confd, False, RUNMODE.Production, 0, None, project_version_file)
     yield
     dodoo._framework = orig_framework
 

diff --git a/dodoo/dodoo/__init__.py b/dodoo/dodoo/__init__.py
@@ -9,53 +9,26 @@
 __all__ = ["RUNMODE", "main", "framework"]
 
 import logging
-import colorlog
-import threading
-import os
+import logging.config
 import enum
 import sys
 
-from pathlib import Path
 
-try:
-    from pythonjsonlogger import jsonlogger
-except ImportError:
-    jsonlogger = None
+from pathlib import Path
+from typing import Optional
+from dodoo.logger import DEFAULT_LOGGING_CONFIG
 
+_log = logging.getLogger(__name__)
 
 assert sys.version_info >= (3, 7), "doDoo requires Python >= 3.7 to run."
 
-_log = logging.getLogger(__name__)
-
 
 class RUNMODE(enum.Enum):
     Develop = 1
     Staging = 2
     Production = 3
 
 
-class DBFormatter(logging.Formatter):
-    def format(self, record):
-        record.pid = os.getpid()
-        record.dbname = getattr(threading.current_thread(), "dbname", "?")
-        return super().format(record)
-
-
-class ColoredFormatter(DBFormatter, colorlog.ColoredFormatter):
-    pass
-
-
-if jsonlogger:
-
-    class JSONFormatter(DBFormatter, jsonlogger.JsonFormatter):
-        def format(self, record):
-            if record.exc_info:
-                mod_name = record.exc_info[0].__module__
-                obj_name = record.exc_info[0].__name__
-                record.exc_class = f"{mod_name}:{obj_name}"
-            return super().format(record)
-
-
 # Those might import RUNMODE
 from .configs import load_config  # noqa
 from .interfaces import odoo  # noqa
@@ -74,16 +47,24 @@ def main(
     config_dir: Path,
     call_home: bool,
     run_mode: RUNMODE,
-    log_level: int,
+    verbosity: Optional[int],
+    log_config: Optional[Path],
     projectversion_file: Path,
 ) -> None:
     """Provide the common cli entrypoint, initialize the dodoo python
     environment, load configuration and set up logging.
 
     Then hand off to a subcommand."""
-    rootlogger = logging.getLogger()
-    rootlogger.setLevel(logging.WARNING - min(log_level, 2) * 10)
+    if verbosity:
+        log_level = logging.WARNING - min(verbosity, 2) * 10
+        logging.getLogger("").setLevel(log_level)
+        logging.getLogger("odoo").setLevel(log_level)
+    if log_config:
+        logging.config.fileConfig(log_config)
+    else:
+        logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
     logging.captureWarnings(True)
+
     config = load_config(config_dir, run_mode)
 
     # Load odoo module from specified framework path
@@ -101,38 +82,6 @@ def main(
 
     odoo.Tools().resetlocale()
 
-    # TODO: Review if this is optimal
-    if run_mode == RUNMODE.Develop:
-        logformat = (
-            "%(levelname)s %(dbname)s "
-            "%(name)s: %(message)s\n"
-            "file://%(pathname)s:%(lineno)d"
-        )
-    else:
-        logformat = "%(asctime)s %(levelname)s %(dbname)s " "%(name)s: %(message)s"
-    handler = colorlog.StreamHandler()
-    if handler.stream.isatty():
-        formatter = ColoredFormatter(
-            logformat,
-            log_colors={
-                "DEBUG": "cyan",
-                "INFO": "green",
-                "WARNING": "yellow",
-                "ERROR": "red",
-                "CRITICAL": "red,bg_white",
-            },
-            secondary_log_colors={"message": {"ERROR": "red", "CRITICAL": "red"}},
-        )
-    elif jsonlogger:
-        formatter = JSONFormatter(logformat)
-    else:
-        formatter = DBFormatter(logformat)
-
-    handler.setFormatter(formatter)
-    logging.getLogger().addHandler(handler)
-
-    config.Odoo.apply_log_handler_to(logging)
-
     if call_home:
         Patcher.features.update(call_home=True)
     Patcher(config.Odoo, config.Db, config.Smtp).apply()

diff --git a/dodoo/dodoo/cli.py b/dodoo/dodoo/cli.py
@@ -40,6 +40,14 @@
     help="Config directory containing json config files for develop, staging & "
     "production. A sane default config is shipped for each env.",
 )
+@click.option(
+    "--log-conf",
+    "log_config",
+    default=False,
+    show_default=True,
+    type=click_pathlib.Path(exists=True, file_okay=False, resolve_path=True),
+    help="Logger configuration file.",
+)
 @click.option(
     "-h",
     "--call-home",
@@ -78,7 +86,7 @@
 @click.option(
     "-v",
     "--verbose",
-    "log_level",
+    "verbosity",
     count=True,
     help="Specify the log level from: info, debug. Without: warn.",
 )

diff --git a/dodoo/dodoo/configs/odoo.py b/dodoo/dodoo/configs/odoo.py
@@ -64,8 +64,6 @@ class OdooConfig(BaseConfig):
     geoip_database:       PathLike = Path("/usr/share/GeoIP/GeoLite2-City.mmdb")  # noqa: E241
     unaccent:                 bool = True  # noqa: E241
     server_wide_modules: FrozenSet[str] = frozenset(["web", "base"])  # noqa: E241
-    log_handler:         FrozenSet[str] = frozenset(["odoo.http.rpc.request:INFO",  # noqa: E241
-                                                "odoo.http.rpc.response:INFO", ])
 
     # Secrets
     class Sec():
@@ -82,14 +80,6 @@ def session_encryption_key(self):
     Sec = Sec()
     # fmt: on
 
-    def apply_log_handler_to(self, logging):
-        for element in self.log_handler:
-            loggername, level = element.split(":")
-            level = getattr(logging, level, logging.INFO)
-            logger = logging.getLogger(loggername)
-            logger.setLevel(level)
-            _log.debug(f"logger level set: {element}")
-
     def apply(self):
         cfg = odoo.Config()
         for k, v in self.__dict__.items():
@@ -102,7 +92,6 @@ def apply(self):
         )
         # Fix frozenset config value
         cfg.config["server_wide_modules"] = ",".join(cfg.config["server_wide_modules"])
-        cfg.config["log_handler"] = ",".join(cfg.config["log_handler"])
         # Fix Path objects
         cfg.config["data_dir"] = str(cfg.config["data_dir"])
         cfg.config["backup_dir"] = str(cfg.config["backup_dir"])

diff --git a/dodoo/dodoo/logger.py b/dodoo/dodoo/logger.py
@@ -0,0 +1,100 @@
+import logging
+import sys
+import threading
+
+import click
+
+DEFAULT_LOGGING_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "default": {
+            "()": "dodoo.logger.DefaultFormatter",
+            "fmt": "%(levelprefix)s %(message)s",
+        },
+        "odoo": {
+            "()": "dodoo.logger.DefaultFormatter",
+            "fmt": "%(levelprefix)s %(message)s  -- %(dbname)s, %(name)s",
+        },
+        # "json": {
+        #     "()": "pythonjsonlogger.jsonlogger.JsonFormatter",
+        # },
+        # "filelink": {
+        #     "()": "dodoo.logger.DefaultFormatter",
+        #     "fmt": (
+        #         "%(levelprefix)s %(dbname)s | %(message)s  (%(name)s)\n"
+        #         "\tfile://%(pathname)s:%(lineno)d"
+        #     ),
+        # },
+    },
+    "handlers": {
+        "default": {
+            "formatter": "default",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stderr",
+        },
+        "odoo": {
+            "formatter": "odoo",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stderr",
+        },
+        # "filelink": {
+        #     "formatter": "filelink",
+        #     "class": "logging.StreamHandler",
+        #     "stream": "ext://sys.stderr",
+        # },
+    },
+    "loggers": {
+        "": {"handlers": ["default"], "level": "INFO"},
+        "odoo": {"handlers": ["odoo"], "level": "INFO", "propagate": False},
+    },
+}
+
+
+class ColourizedFormatter(logging.Formatter):
+    """
+    A custom log formatter class that:
+    * Outputs the LOG_LEVEL with an appropriate color.
+    """
+
+    level_name_colors = {
+        logging.DEBUG: lambda level_name: click.style(str(level_name), fg="cyan"),
+        logging.INFO: lambda level_name: click.style(str(level_name), fg="green"),
+        logging.WARNING: lambda level_name: click.style(str(level_name), fg="yellow"),
+        logging.ERROR: lambda level_name: click.style(str(level_name), fg="red"),
+        logging.CRITICAL: lambda level_name: click.style(
+            str(level_name), fg="bright_red"
+        ),
+    }
+
+    def __init__(self, fmt=None, datefmt=None, style="%"):
+        self.use_colors = self.should_use_colors()
+        super().__init__(fmt=fmt, datefmt=datefmt, style=style)
+
+    def color_level_name(self, level_name, level_no):
+        default = lambda level_name: str(level_name)  # noqa: E731
+        func = self.level_name_colors.get(level_no, default)
+        return func(level_name)
+
+    def should_use_colors(self):
+        return True
+
+    def formatMessage(self, record):
+        levelname = record.levelname
+        seperator = " " * (8 - len(record.levelname))
+        dbname = getattr(threading.current_thread(), "dbname", "***") or "***"
+        name = record.name
+        if self.use_colors:
+            levelname = self.color_level_name(levelname, record.levelno)
+            dbname = click.style(dbname, fg="blue")
+            name = click.style(name, bold=True)
+
+        record.__dict__["levelprefix"] = levelname + ":" + seperator
+        record.__dict__["dbname"] = dbname
+        record.__dict__["name"] = name
+        return super().formatMessage(record)
+
+
+class DefaultFormatter(ColourizedFormatter):
+    def should_use_colors(self):
+        return sys.stderr.isatty()
diff --git a/dodoo/tests/utils/conftest.py b/dodoo/tests/utils/conftest.py
@@ -14,7 +14,7 @@ def main_loaded(confd, project_version_file, framework_dir, mocker):
 
     orig_framework = dodoo._framework
     mocker.patch("dodoo.create_custom_schema_layout")
-    main(framework_dir, confd, False, RUNMODE.Production, 0, project_version_file)
+    main(framework_dir, confd, False, RUNMODE.Production, 0, None, project_version_file)
     yield
     dodoo._framework = orig_framework