feat: add loggers package

packages-backend/express/README.md

@@ -51,7 +51,7 @@ app.use((request, response, next) => {
 
 - `inferIssuer` (_boolean_): Determines whether the issuer should be inferred from the custom request HTTP header `x-mc-api-cloud-identifier` which is sent by the Merchant Center API Gateway when forwarding the request. This might be useful in case the server is used in multiple regions.
 
-- `jwks` (_object_): see options of `jwks-rsa`.
+- `jwks` (_object_): See options of `jwks-rsa`.
 
 ### Usage in Serverless Functions
 

packages-backend/loggers/.gitignore

@@ -0,0 +1 @@
+build

packages-backend/loggers/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 commercetools GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages-backend/loggers/README.md

@@ -0,0 +1,73 @@
+# @commercetools-backend/loggers
+
+<p align="center">
+  <a href="https://www.npmjs.com/package/@commercetools-backend/loggers"><img src="https://badgen.net/npm/v/@commercetools-backend/loggers" alt="Latest release (latest dist-tag)" /></a> <a href="https://www.npmjs.com/package/@commercetools-backend/loggers"><img src="https://badgen.net/npm/v/@commercetools-backend/loggers/next" alt="Latest release (next dist-tag)" /></a> <a href="https://bundlephobia.com/result?p=@commercetools-backend/loggers"><img src="https://badgen.net/bundlephobia/minzip/@commercetools-backend/loggers" alt="Minified + GZipped size" /></a> <a href="https://github.com/commercetools/merchant-center-application-kit/blob/master/LICENSE"><img src="https://badgen.net/github/license/commercetools/merchant-center-application-kit" alt="GitHub license" /></a>
+</p>
+
+Opinionated JSON loggers for HTTP server applications.
+
+## Install
+
+```bash
+$ npm install --save @commercetools-backend/loggers
+```
+
+## Access logger
+
+Creates a logger to be used for HTTP requests access logs.
+
+```js
+const { createAccessLogger } = require('@commercetools-backend/loggers');
+
+app.use(createAccessLogger());
+```
+
+### Access logger options
+
+- `ignoreUrls` (_Array of string_): A list of URL paths to be ignored from being logged.
+
+## Application logger
+
+Creates a logger to be used programmatically in the application code.
+
+```js
+const { createApplicationLogger } = require('@commercetools-backend/loggers');
+
+const app = createApplicationLogger();
+
+app.info('Hey there', { meta: { name: 'Tom' } });
+```
+
+## Error report logger (Sentry)
+
+Creates a logger to be used for error reporting with Sentry.
+
+```js
+const { createErrorReportLogger } = require('@commercetools-backend/loggers');
+
+const { sentryRequestHandler } = createErrorReportLogger();
+
+app.use(sentryRequestHandler);
+```
+
+```js
+const { createErrorReportLogger } = require('@commercetools-backend/loggers');
+
+const { trackError } = createErrorReportLogger();
+
+trackError(error, { request }, (errorId) => {
+  if (errorId) {
+    // Attach the Sentry error id to the custom response header
+    response.setHeader('X-Sentry-Error-Id', errorId);
+  }
+  response.end();
+});
+```
+
+### Error report logger options
+
+- `sentry` (_object_): An optional configuration object for Sentry.
+  - `sentry.DSN` (_string_): The DSN value of your Sentry project.
+  - `sentry.role` (_string_): The value for the `role` Sentry tag.
+  - `sentry.environment` (_string_): The value for the `environment` Sentry tag.
+- `errorMessageBlacklist` (_Array of string or RegExp_): A list of error messages for which the error should not be reported, if the error message matches.

packages-backend/loggers/index.ts

@@ -0,0 +1,5 @@
+// This file exists  because we want jest to use our non-compiled code to run tests
+// if this file is missing, and you have a `module` or `main` that points to a non-existing file
+// (ie, a bundle that hasn't been built yet) then jest will fail if the bundle is not yet built.
+// all apps should export all their named exports from their root index.js
+export * from './src';

packages-backend/loggers/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@commercetools-backend/loggers",
+  "version": "1.0.0",
+  "description": "Opinionated JSON loggers for HTTP server applications",
+  "bugs": "https://github.com/commercetools/merchant-center-application-kit/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/commercetools/merchant-center-application-kit.git",
+    "directory": "packages-backend/loggers"
+  },
+  "homepage": "https://docs.commercetools.com/custom-applications",
+  "keywords": ["javascript", "nodejs", "express", "logger", "server", "toolkit"],
+  "license": "MIT",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "./build/index.js",
+  "typings": "./build/index.d.ts",
+  "types": "./build/index.d.ts",
+  "files": ["build", "package.json", "LICENSE", "README.md"],
+  "scripts": {
+    "prebuild": "rimraf build/**",
+    "build": "tsc -p tsconfig.build.json"
+  },
+  "dependencies": {
+    "@sentry/node": "5.15.5",
+    "@types/triple-beam": "1.3.0",
+    "express-winston": "4.0.3",
+    "fast-safe-stringify": "2.0.7",
+    "lodash": "4.17.15",
+    "logform": "2.1.2",
+    "triple-beam": "1.3.0",
+    "winston": "3.2.1"
+  },
+  "devDependencies": {
+    "express": "4.17.1"
+  }
+}

packages-backend/loggers/src/create-access-logger.ts

@@ -0,0 +1,51 @@
+import type { RequestFilter } from 'express-winston';
+
+import expressWinston from 'express-winston';
+import winston from 'winston';
+import * as env from './env';
+import redactInsecureRequestHeaders from './utils/redact-insecure-request-headers';
+import jsonFormatter from './custom-formats/json';
+
+const shouldLog = !env.isTest || process.env.DEBUG === 'true';
+
+type LoggerOptions = {
+  ignoreUrls?: string[];
+};
+
+const defaultIgnoreUrls = ['/', '/health', '/favicon.ico'];
+const defaultFormat = winston.format.combine(winston.format.timestamp());
+
+// Inspired by https://github.com/bithavoc/express-winston/issues/62#issuecomment-396906056
+const requestFilter: RequestFilter = (req, propName) => {
+  if (propName === 'headers') {
+    return redactInsecureRequestHeaders(req);
+  }
+  return req[propName];
+};
+
+const createAccessLogger = (options: LoggerOptions = {}) => {
+  const ignoreUrls = [...defaultIgnoreUrls, ...(options.ignoreUrls ?? [])];
+
+  return expressWinston.logger({
+    transports: [new winston.transports.Console()],
+    format: winston.format.combine(
+      defaultFormat,
+      env.isDev ? winston.format.cli() : jsonFormatter()
+    ),
+    requestFilter,
+    meta: true,
+    expressFormat: true, // Use default morgan access log formatting
+    colorize: env.isDev,
+    skip: (req) => !shouldLog || ignoreUrls.includes(req.originalUrl),
+    dynamicMeta: (req) => ({
+      ip: req.ip,
+      ips: req.ips,
+      hostname: req.hostname,
+      ...(req.connection.remoteAddress
+        ? { remoteAddress: req.connection.remoteAddress }
+        : {}),
+    }),
+  });
+};
+
+export default createAccessLogger;

packages-backend/loggers/src/create-application-logger.ts

@@ -0,0 +1,21 @@
+import winston from 'winston';
+import * as env from './env';
+import jsonFormatter from './custom-formats/json';
+
+const shouldLog = !env.isTest || process.env.DEBUG === 'true';
+
+const createApplicationLogger = () => {
+  return winston.createLogger({
+    level: process.env.DEBUG === 'true' ? 'debug' : 'info',
+    format: env.isDev
+      ? winston.format.combine(winston.format.cli(), winston.format.simple())
+      : jsonFormatter(),
+    transports: [
+      new winston.transports.Console({
+        silent: !shouldLog,
+      }),
+    ],
+  });
+};
+
+export default createApplicationLogger;

packages-backend/loggers/src/create-error-report-logger.ts

@@ -0,0 +1,89 @@
+import type { Request, Response, NextFunction } from 'express';
+
+import * as Sentry from '@sentry/node';
+import * as env from './env';
+import redactInsecureRequestHeaders from './utils/redact-insecure-request-headers';
+
+const shouldLog = !env.isTest || process.env.DEBUG === 'true';
+
+type LoggerOptions = {
+  sentry?: {
+    DSN: string;
+    role: string;
+    environment: string;
+  };
+  errorMessageBlacklist?: Array<string | RegExp>;
+};
+
+function createErrorReportLogger(options: LoggerOptions = {}) {
+  // Sentry
+  if (options.sentry) {
+    Sentry.init({ dsn: options.sentry.DSN });
+    Sentry.configureScope((scope) => {
+      scope.setLevel(Sentry.Severity.Error);
+    });
+    Sentry.setTag('role', options.sentry.role);
+    Sentry.setTag('environment', options.sentry.environment);
+  }
+
+  // Filter out errors that contains those strings either as name or message
+  const errorMessageBlacklist = options.errorMessageBlacklist ?? [];
+
+  const shouldErrorBeTracked = (error: Error) =>
+    !errorMessageBlacklist.some(
+      // The match can be either the error code as well as a part of
+      // the error message (in case the error code is not enough).
+      // Since it can be a mix of both, we need to match it
+      // to either the name or message.
+      (match) => {
+        if (typeof match === 'string') {
+          return match === error.name || match === error.message;
+        }
+        return match.test(error.name) || match.test(error.message);
+      }
+    );
+
+  function trackError(
+    error: Error,
+    meta: { request: Request; userId?: string },
+    getErrorId: (errorId?: string | null) => void
+  ) {
+    if (!shouldLog) {
+      getErrorId(null);
+      return;
+    }
+    if (options.sentry && shouldErrorBeTracked(error)) {
+      if (meta.userId) {
+        Sentry.setUser({ id: meta.userId });
+      }
+      const { method, originalUrl, httpVersion } = meta.request;
+      Sentry.setExtra('request', {
+        method,
+        originalUrl,
+        httpVersion,
+        headers: redactInsecureRequestHeaders(meta.request),
+      });
+      const errorId = Sentry.captureException(error);
+      getErrorId(errorId);
+    } else {
+      getErrorId();
+    }
+  }
+
+  const passThroughMiddleware = (
+    _req: Request,
+    _res: Response,
+    next: NextFunction
+  ) => next();
+
+  function sentryRequestHandler() {
+    if (shouldLog && options.sentry) {
+      return Sentry.Handlers.requestHandler({ request: false });
+    }
+    return passThroughMiddleware;
+  }
+
+  return { sentryRequestHandler, trackError };
+}
+
+export default createErrorReportLogger;

packages-backend/loggers/src/custom-formats/json.ts

@@ -0,0 +1,55 @@
+// This file is an exteded version of the `json` formatter so that we
+// can rename the `level` field.
+import type { TransformableInfo } from 'logform';
+
+import { format } from 'logform';
+import { MESSAGE } from 'triple-beam';
+import jsonStringify from 'fast-safe-stringify';
+import getIn from 'lodash/get';
+import setIn from 'lodash/set';
+import unsetIn from 'lodash/unset';
+
+/*
+ * function replacer (key, value)
+ * Handles proper stringification of Buffer output.
+ */
+function replacer(key: string, value: Buffer | string) {
+  if (key === 'queryJsonString') {
+    // We need to stringify the value as otherwise Kibana cardinality explodes.
+    // Ref: https://commercetools.slack.com/archives/C011FLZ1JLW/p1588066334005900
+    return JSON.stringify(value);
+  }
+  return value instanceof Buffer ? value.toString('base64') : value;
+}
+
+function replaceField(
+  info: TransformableInfo,
+  jsonPath: string,
+  newJsonPath: string
+) {
+  const val = getIn(info, jsonPath);
+  if (val) {
+    unsetIn(info, jsonPath);
+    setIn(info, newJsonPath, val);
+  }
+}
+
+/*
+ * function json (info)
+ * Returns a new instance of the JSON format that turns a log `info`
+ * object into pure JSON. This was previously exposed as { json: true }
+ * to transports in `winston < 3.0.0`.
+ */
+/* eslint-disable no-param-reassign */
+const jsonFormatter = format((info, opts = {}) => {
+  info.logLevel = info.level;
+  delete info.level;
+
+  // Replace / rename fields
+  replaceField(info, 'meta.req.query', 'meta.req.queryJsonString');
+
+  info[MESSAGE] = jsonStringify(info, opts.replacer ?? replacer, opts.space);
+  return info;
+});
+
+export default jsonFormatter;

packages-backend/loggers/src/env.ts

@@ -0,0 +1,6 @@
+const env = process.env.NODE_ENV;
+const isDev = !env || env === 'development';
+const isProd = env === 'production';
+const isTest = env === 'test';
+
+export { isDev, isProd, isTest };

packages-backend/loggers/src/index.ts

@@ -0,0 +1,4 @@
+export { default as createAccessLogger } from './create-access-logger';
+export { default as createApplicationLogger } from './create-application-logger';
+export { default as createErrorReportLogger } from './create-error-report-logger';
+export { default as redactInsecureRequestHeaders } from './utils/redact-insecure-request-headers';