WIP: Pattern Obfuscation

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@krauters/logger",
 	"description": "TypeScript wrapper for Winston, optimized for structured Lambda logs and CloudWatch.",
-	"version": "2.0.0",
+	"version": "2.1.0",
 	"main": "dist/src/index.js",
 	"type": "commonjs",
 	"scripts": {
@@ -40,17 +40,17 @@
 		"nodemon": "^3.1.9",
 		"ts-jest": "^29.2.5",
 		"ts-node": "^10.9.2",
-		"typescript": "^5.7.2"
+		"typescript": "^5.7.3"
 	},
 	"files": [
 		"dist"
 	],
 	"dependencies": {
-		"@aws-sdk/client-cloudwatch": "^3.714.0",
-		"@aws-sdk/client-s3": "^3.714.0",
+		"@aws-sdk/client-cloudwatch": "^3.726.1",
+		"@aws-sdk/client-s3": "^3.726.1",
 		"@krauters/environment": "^0.5.2",
 		"@krauters/structures": "^1.3.0",
-		"@types/aws-lambda": "^8.10.146",
+		"@types/aws-lambda": "^8.10.147",
 		"@types/uuid": "^10.0.0",
 		"nanoid": "^5.0.9",
 		"winston": "^3.17.0"

src/config.ts

@@ -3,101 +3,16 @@ import { EnvironmentBuilder } from '@krauters/environment'
 import { Env, Stage } from '@krauters/structures'
 import { hostname } from 'os'
 
+import type { ConfigOptions } from './structures'
+
 import { empty, LogLevel } from './structures'
 import { isFalsy } from './utils'
 
 export type Config = ReturnType<typeof getConfig>
 
-/**
- * Configuration options for retrieving and building the logger environment configuration.
- */
-export interface ConfigOptions {
-	CODENAME: string
-
-	/**
-	 * The current environment (e.g. Development, Production).
-	 */
-	ENV: Env
-
-	/**
-	 * An optional prefix to apply to environment variables.
-	 */
-	ENVIRONMENT_PREFIX?: string
-
-	/**
-	 * The hostname of the machine or environment running the code.
-	 */
-	HOST: string
-
-	/**
-	 * The output format of logs, either 'friendly' or 'structured'.
-	 */
-	LOG_FORMAT: string
-
-	/**
-	 * Fields to be hidden from friendly log format output.
-	 */
-	LOG_FRIENDLY_FIELDS_HIDE?: string[]
-
-	/**
-	 * The minimum log level at which logs are recorded.
-	 */
-	LOG_LEVEL: LogLevel
-
-	/**
-	 * A string prefix applied to every log message.
-	 */
-	LOG_PREFIX: string
-
-	/**
-	 * A string used to separate sections within a log message.
-	 */
-	LOG_SECTION_SEPARATOR: string
-
-	/**
-	 * Fields to be hidden from structured log format output.
-	 */
-	LOG_STRUCTURED_FIELDS_HIDE?: string[]
-
-	/**
-	 * The package or application name.
-	 */
-	PACKAGE: string
-
-	/**
-	 * If true, values can be pulled from environment variables. Defaults to true if not provided.
-	 */
-	PULL_FROM_ENVIRONMENT?: boolean
-
-	/**
-	 * A static request ID to be used in logs. If not provided, one will be generated.
-	 */
-	REQUEST_ID?: string
-
-	/**
-	 * If true, enables simpler log formatting without full request IDs.
-	 */
-	SIMPLE_LOGS: boolean
-
-	/**
-	 * The current stage or tier (e.g. Beta, Production).
-	 */
-	STAGE: Stage
-
-	/**
-	 * The format of timestamps in log messages.
-	 */
-	TIMESTAMP_FORMAT: string
-
-	/**
-	 * The current version of the application or service.
-	 */
-	VERSION: string
-}
-
 /**
  * Retrieves the application's environment configuration, allowing overrides via an options object.
- * If pullFromEnv is false, will not load values from environment variables.
+ * If PULL_FROM_ENVIRONMENT is false, will not load values from environment variables.
  *
  * @param options Optional configuration overrides.
  * @returns The environment variable configuration.
@@ -115,14 +30,16 @@ export function getConfig(options?: Partial<ConfigOptions>) {
 		'LOG_LEVEL',
 		'LOG_PREFIX',
 		'LOG_SECTION_SEPARATOR',
+		'OBFUSCATION_ENABLED',
+		'OBFUSCATION_PATTERNS',
 		'PACKAGE',
 		'SIMPLE_LOGS',
 		'STAGE',
 		'TIMESTAMP_FORMAT',
 		'VERSION',
 	)
-		.optionals('LOG_FRIENDLY_FIELDS_HIDE', 'LOG_STRUCTURED_FIELDS_HIDE', 'REQUEST_ID')
-		.transform((value) => !isFalsy(value), 'SIMPLE_LOGS')
+		.optionals('LOG_FRIENDLY_FIELDS_HIDE', 'LOG_STRUCTURED_FIELDS_HIDE', 'REQUEST_ID', 'LOG_PROCESSOR')
+		.transform((value) => !isFalsy(value), 'SIMPLE_LOGS', 'OBFUSCATION_ENABLED')
 		.transform(
 			(value) => (value.trim() === '' ? [] : value.replace(/\s/g, '').split(',')),
 			'LOG_FRIENDLY_FIELDS_HIDE',
@@ -131,15 +48,24 @@ export function getConfig(options?: Partial<ConfigOptions>) {
 		.transform((value) => value as Env, 'ENV')
 		.transform((value) => value as Stage, 'STAGE')
 		.transform((value) => value as LogLevel, 'LOG_LEVEL')
+		.transform(
+			(value) => (typeof value === 'string' && value.trim() !== '' ? [new RegExp(value.trim())] : []),
+			'OBFUSCATION_PATTERNS',
+		)
 		.defaults({
 			// Actual Defaults
 			CODENAME: empty,
+			CUSTOM_LOG_PROCESSOR: undefined,
 			ENV: Env.Unknown,
 			HOST: hostname(),
 			LOG_FORMAT: 'friendly',
 			LOG_LEVEL: LogLevel.Info,
 			LOG_PREFIX: '',
 			LOG_SECTION_SEPARATOR: ' | ',
+			OBFUSCATION_ENABLED: true,
+
+			// Object.values(SensitivePatterns).map((pattern) => new RegExp(pattern))
+			OBFUSCATION_PATTERNS: [],
 			PACKAGE: empty,
 			SIMPLE_LOGS: false,
 			STAGE: Stage.Unknown,

src/index.ts

@@ -1,4 +1,5 @@
 export * from './config'
 export * from './container'
 export type * from './logger'
+export * from './sensitive-patterns'
 export * from './structures'

src/logger.ts

@@ -11,7 +11,14 @@ import { v4 as uuidv4 } from 'uuid'
 import { addColors, createLogger, format, transports, config as winstonConfig } from 'winston'
 
 import type { Config } from './config'
-import type { GetLogObjectParams, LoggerOptions, LogOptions, Metadata, PublishMetricOptions } from './structures'
+import type {
+	GetLogObjectParams,
+	LoggerOptions,
+	LogOptions,
+	LogProcessor,
+	Metadata,
+	PublishMetricOptions,
+} from './structures'
 
 import { getConfig } from './config'
 import { empty, LogLevel } from './structures'
@@ -23,9 +30,10 @@ export class Logger {
 	public metadata: Metadata = {}
 	private colors: Record<string, string | string[]>
 	private levels: Record<string, number>
+	private readonly MASK: string = '****MASKED****'
 
 	public constructor(options: LoggerOptions = {}) {
-		const { configOptions, context, format: customFormat, transports: customTransports } = options
+		const { configOptions, context, format: customFormat, logProcessor, transports: customTransports } = options
 		this.config = getConfig(configOptions)
 
 		const requestId = this.getRequestId(context)
@@ -37,7 +45,7 @@ export class Logger {
 		addColors(this.colors)
 
 		this.logger = createLogger({
-			format: customFormat ?? this.getFormatter(),
+			format: customFormat ?? this.getFormatter(logProcessor),
 			level: this.config.LOG_LEVEL,
 			levels: this.levels,
 			transports: customTransports ?? [new transports.Console()],
@@ -71,10 +79,10 @@ export class Logger {
 		].join(separator)
 	}
 
-	public getFormatter(): Format {
+	public getFormatter(logProcessor?: LogProcessor): Format {
 		const formatters = {
-			friendly: this.getFriendlyFormat.bind(this),
-			structured: this.getStructuredFormat.bind(this),
+			friendly: this.getFriendlyFormat.bind(this, logProcessor),
+			structured: this.getStructuredFormat.bind(this, logProcessor),
 		}
 
 		const formatType = this.config.LOG_FORMAT as keyof typeof formatters
@@ -85,21 +93,27 @@ export class Logger {
 		return formatters[formatType]()
 	}
 
-	public getFriendlyFormat(): Format {
+	public getFriendlyFormat(logProcessor?: LogProcessor): Format {
 		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 		const separator: string = this.config.LOG_SECTION_SEPARATOR!
 		const prefix: string = this.config.LOG_PREFIX ?? ''
 
 		return format.combine(
 			format.colorize({ all: true }),
 			format.timestamp({ format: this.config.TIMESTAMP_FORMAT }),
-			format.printf((info) =>
-				this.formatLogMessage(
-					this.getLogObject({ fieldsToHide: this.config.LOG_FRIENDLY_FIELDS_HIDE, info }),
-					separator,
-					prefix,
-				),
-			),
+			format.printf((info) => {
+				let logObject = this.getLogObject({ fieldsToHide: this.config.LOG_FRIENDLY_FIELDS_HIDE, info })
+
+				if (this.config.OBFUSCATION_ENABLED && this.config.OBFUSCATION_PATTERNS) {
+					logObject = this.applyObfuscation(logObject)
+				}
+
+				if (logProcessor) {
+					logObject = logProcessor(logObject)
+				}
+
+				return this.formatLogMessage(logObject, separator, prefix)
+			}),
 		)
 	}
 
@@ -115,19 +129,29 @@ export class Logger {
 	}
 
 	public getRequestId(context?: LambdaContext): string {
-		if (this.config?.REQUEST_ID) return this.config.REQUEST_ID
+		if (this.config.REQUEST_ID) return this.config.REQUEST_ID
 		if (context?.awsRequestId) return context.awsRequestId
 		if (this.config.SIMPLE_LOGS) return uuidv4().split('-')[0]
 
 		return uuidv4()
 	}
 
-	public getStructuredFormat(): Format {
+	public getStructuredFormat(logProcessor?: LogProcessor): Format {
 		return format.combine(
 			format.timestamp({ format: this.config.TIMESTAMP_FORMAT }),
-			format.printf((info) =>
-				JSON.stringify(this.getLogObject({ fieldsToHide: this.config.LOG_STRUCTURED_FIELDS_HIDE, info })),
-			),
+			format.printf((info) => {
+				let logObject = this.getLogObject({ fieldsToHide: this.config.LOG_STRUCTURED_FIELDS_HIDE, info })
+
+				if (this.config.OBFUSCATION_ENABLED && this.config.OBFUSCATION_PATTERNS) {
+					logObject = this.applyObfuscation(logObject)
+				}
+
+				if (logProcessor) {
+					logObject = logProcessor(logObject)
+				}
+
+				return JSON.stringify(logObject)
+			}),
 		)
 	}
 
@@ -180,7 +204,7 @@ export class Logger {
 	}
 
 	public updateInstance(options: LoggerOptions): void {
-		const { configOptions, context, format: customFormat, transports: customTransports } = options
+		const { configOptions, context, format: customFormat, logProcessor, transports: customTransports } = options
 
 		if (configOptions) {
 			this.config = getConfig(configOptions)
@@ -195,21 +219,22 @@ export class Logger {
 
 			this.metadata = { ...this.getBaseMetadata(newRequestId), ...userFields }
 
-			this.logger.format = this.getFormatter()
+			this.logger.format = this.getFormatter(logProcessor)
 
 			// Explicitly update log level
 			this.logger.level = this.config.LOG_LEVEL as string
 		} else if (context) {
 			const newRequestId = this.getRequestId(context)
 			const userFields = { ...this.metadata }
+
 			for (const key of Object.keys(this.getBaseMetadata(newRequestId))) {
 				// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
 				delete userFields[key]
 			}
 
 			this.metadata = { ...this.getBaseMetadata(newRequestId), ...userFields }
 
-			this.logger.format = this.getFormatter()
+			this.logger.format = this.getFormatter(logProcessor)
 		}
 
 		if (customFormat) {
@@ -222,6 +247,10 @@ export class Logger {
 				this.logger.add(transport)
 			}
 		}
+
+		if (options.logProcessor) {
+			this.logger.format = this.getFormatter(options.logProcessor)
+		}
 	}
 
 	public updateLevels(newLevels: Record<string, number>, newColors: Record<string, string>): void {
@@ -246,6 +275,26 @@ export class Logger {
 		this.log({ level: LogLevel.Warn, message, metadata: data })
 	}
 
+	/**
+	 * Applies obfuscation to the log object based on configured regex patterns.
+	 *
+	 * @param logObject The original log object.
+	 * @returns The obfuscated log object.
+	 */
+	private applyObfuscation(logObject: Record<string, unknown>): Record<string, unknown> {
+		const obfuscated = { ...logObject }
+
+		for (const key in obfuscated) {
+			if (typeof obfuscated[key] === 'string') {
+				for (const pattern of this.config.OBFUSCATION_PATTERNS ?? []) {
+					obfuscated[key] = (obfuscated[key] as string).replace(pattern, this.MASK)
+				}
+			}
+		}
+
+		return obfuscated
+	}
+
 	private getBaseMetadata(requestId: string): Metadata {
 		const base = {
 			codename: this.config.CODENAME,