Skip to content

Commit

Permalink
DPO3DPKRT-863/Slack Notifications (#634)
Browse files Browse the repository at this point in the history 
(new) notifySlack class and access via recordKeeper
(new) Slack API key is stored as an environment variable
(new) styled buttons for messages with links. uses error/success colors
(new) compact messages and details placed as a reply
(new) cleanChannel utility to empty an existing channel
(new) sendMessage to send to both email & slack
(new) slack message testing routines
(new) exposed rate manager objects so can get config and metrics
(new) random NotifyPackage routine for testing emails & slack

(fix) changed NotifyChannel to NotifyUserGroup for clarity
(fix) relocated testing routines into Notify class
(fix) catch undefined errors when flatten objects in log
  • Loading branch information
@EMaslowskiQ
EMaslowskiQ authored Oct 18, 2024
1 parent 4d720cb commit dfb1a68
Show file tree
Hide file tree
Showing 9 changed files with 784 additions and 228 deletions.
6 changes: 6 additions & 0 deletions server/config/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ export type ConfigType = {
navigation: {
type: NAVIGATION_TYPE;
},
slack: {
apiKey: string;
},
storage: {
type: STORAGE_TYPE;
rootRepository: string;
Expand Down Expand Up @@ -203,6 +206,9 @@ export const Config: ConfigType = {
navigation: {
type: NAVIGATION_TYPE.SOLR,
},
slack: {
apiKey: process.env.PACKRAT_SLACK_KEY ? process.env.PACKRAT_SLACK_KEY: 'undefined',
},
storage: {
type: STORAGE_TYPE.LOCAL,
rootRepository: process.env.PACKRAT_OCFL_STORAGE_ROOT ? process.env.PACKRAT_OCFL_STORAGE_ROOT : /* istanbul ignore next */ './var/Storage/Repository',
Expand Down
6 changes: 5 additions & 1 deletion server/http/routes/sandbox.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Request, Response } from 'express';
import * as H from '../../utils/helpers';
import { RecordKeeper as RK } from '../../records/recordKeeper';
// import { SlackChannel } from '../../records/notify/notifySlack';

export const play = async (_req: Request, res: Response): Promise<void> => {

Expand All @@ -11,7 +12,10 @@ export const play = async (_req: Request, res: Response): Promise<void> => {
// const result = await RK.logTest(numLogs);

// test email notifications
const result = await RK.emailTest(5);
// const result = await RK.emailTest(5);

// test slack notifications
const result = await RK.slackTest(30,true,RK.SlackChannel.PACKRAT_OPS);

// return our results
res.status(200).send(H.Helpers.JSONStringify(result));
Expand Down
31 changes: 13 additions & 18 deletions server/records/logger/log.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { createLogger, format, transports, addColors } from 'winston';
import * as path from 'path';
import * as fs from 'fs';
import { RateManager, RateManagerConfig, RateManagerResult } from '../utils/rateManager';
import { ENVIRONMENT_TYPE } from '../../config';

// adjust our default event hanlder to support higher throughput. (default is 10)
require('events').EventEmitter.defaultMaxListeners = 50;
Expand Down Expand Up @@ -62,7 +63,7 @@ type DataType = string | number | boolean | object | any[]; // valid types for o
interface LoggerContext {
section: string | null;
caller: string | null;
environment: 'prod' | 'dev';
environment: 'production' | 'development';
idUser: number | undefined;
idRequest: number | undefined;
}
Expand All @@ -89,7 +90,7 @@ interface LoggerResult extends RateManagerResult {}
export class Logger {
private static logger: any | null = null;
private static logDir: string = path.join(__dirname, 'Logs');
private static environment: 'prod' | 'dev' = 'dev';
private static environment: ENVIRONMENT_TYPE = ENVIRONMENT_TYPE.DEVELOPMENT;
private static requests: Map<string, ProfileRequest> = new Map<string, ProfileRequest>();
private static stats: LoggerStats = {
counts: { profile: 0, critical: 0, error: 0, warning: 0, info: 0, debug: 0, total: 0 },
Expand All @@ -104,10 +105,10 @@ export class Logger {
// we're initialized if we have a logger running
return (Logger.logger);
}
public static configure(logDirectory: string, env: 'prod' | 'dev', rateManager: boolean = true, targetRate?: number, burstRate?: number, burstThreshold?: number): LoggerResult {
public static configure(logDirectory: string, environment: ENVIRONMENT_TYPE, rateManager: boolean = true, targetRate?: number, burstRate?: number, burstThreshold?: number): LoggerResult {
// we allow for re-assigning configuration options even if already running
Logger.logDir = logDirectory;
Logger.environment = env;
Logger.environment = environment;

// if we want a rate limiter then we build it
if(rateManager===true) {
Expand Down Expand Up @@ -237,31 +238,23 @@ export class Logger {
Logger.logger = createLogger({
level: 'perf', // Logging all levels
levels: customLevels.levels,
transports: env === 'dev' ? [fileTransport, consoleTransport] : [fileTransport],
transports: environment===ENVIRONMENT_TYPE.DEVELOPMENT ? [fileTransport, consoleTransport] : [fileTransport],
// exitOnError: false, // do not exit on exceptions. combines with 'handleExceptions' above
});

// add our custom colors as well
addColors(customLevels.colors);

// start our rate manager if needed
// if(Logger.rateManager)
// Logger.rateManager.startRateManager();

// start up our metrics tracker (sampel every 5 seconds, 10 samples per avgerage calc)
Logger.trackLogMetrics(5000,10);
} catch(error) {

// if(Logger.rateManager)
// Logger.rateManager.stopRateManager();

return {
success: false,
message: error instanceof Error ? error.message : String(error)
};
}

return { success: true, message: `configured Logger. Sending to file ${(env==='dev') ? 'and console' : ''}` };
return { success: true, message: `configured Logger. Sending to file ${(environment===ENVIRONMENT_TYPE.DEVELOPMENT) ? 'and console' : ''}` };
}
public static getStats(): LoggerStats {
Logger.stats.counts.total = (Logger.stats.counts.critical + Logger.stats.counts.error + Logger.stats.counts.warning + Logger.stats.counts.info + Logger.stats.counts.debug);
Expand Down Expand Up @@ -334,12 +327,14 @@ export class Logger {
private static flattenObject(obj: object, prefix = ''): Record<string, string> {
return Object.keys(obj).reduce((acc, key) => {
const newKey = prefix ? `${prefix}.${key}` : key; // Handle nested keys with dot notation
const value = (obj as Record<string, any>)[key];
let value = (obj as Record<string, any>)[key];

if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
if (typeof value === 'object' && value !== null && value !== undefined && !Array.isArray(value)) {
Object.assign(acc, Logger.flattenObject(value, newKey)); // Recursively flatten nested objects
} else {
acc[newKey] = value.toString(); // Assign non-object values directly
if(newKey==='error' && !value)
value = 'undefined error';
acc[newKey] = value?.toString(); // Assign non-object values directly
}

return acc;
Expand Down Expand Up @@ -655,7 +650,7 @@ export class Logger {
context: {
section: randomSection,
caller: `${String(index).padStart(5,'0')} - `+randomCaller,
environment: Math.random() < 0.5 ? 'prod' : 'dev',
environment: Math.random() < 0.5 ? 'production' : 'development',
idUser: Math.random() < 0.5 ? Math.floor(100 + Math.random() * 900) : undefined,
idRequest: Math.random() < 0.5 ? Math.floor(Math.random() * 100000) : undefined,
}
Expand Down
109 changes: 104 additions & 5 deletions server/records/notify/notify.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { NotifyResult, NotifyPackage, NotifyChannel, NotifyType } from './notifyShared';
import { ENVIRONMENT_TYPE } from '../../config';
import { NotifyResult, NotifyPackage, NotifyUserGroup, NotifyType, randomNotifyPackage } from './notifyShared';
import { NotifyEmail } from './notifyEmail';
import { NotifySlack, SlackChannel } from './notifySlack';
import { Logger as LOG, LogSection } from '../logger/log';

export class Notify {
// wrapper class for email and slack notifications to unify types and methods

//#region EMAIL
public static configureEmail(env: 'prod' | 'dev', targetRate?: number, burstRate?: number, burstThreshold?: number): NotifyResult {
public static configureEmail(env: ENVIRONMENT_TYPE, targetRate?: number, burstRate?: number, burstThreshold?: number): NotifyResult {
return NotifyEmail.configure(env,targetRate,burstRate,burstThreshold);
}

Expand All @@ -14,12 +18,107 @@ export class Notify {
public static sendEmailMessageRaw = NotifyEmail.sendMessageRaw as (type: NotifyType, sendTo: string[], subject: string, textBody: string, htmlBody?: string) => Promise<NotifyResult>;

// testing emails
public static testEmail = NotifyEmail.testEmails as (numEmails: number) => Promise<NotifyResult>;
public static async testEmail(numEmails: number): Promise<NotifyResult> {

const rateManager = NotifyEmail.getRateManager();
const hasRateManager: boolean = rateManager ? true : false;
const config = rateManager?.getConfig();

// create our profiler
// we use a random string in case another test or profile is run to avoid collisisons
const profileKey: string = `EmailTest_${Math.random().toString(36).substring(2, 6)}`;
await LOG.profile(profileKey, LogSection.eSYS, `Email test: ${new Date().toLocaleString()}`, {
numEmails,
rateManager: hasRateManager,
...(hasRateManager === true && config && {
config: (({ onPost: _onPost, ...rest }) => rest)(config) // Exclude onPost
})
},'Notify.testEmails');

// test our slack
const errors: string[] = [];
const promises: Promise<NotifyResult>[] = [];

for(let i=0; i<numEmails; ++i) {
const emailPackage: NotifyPackage = randomNotifyPackage(i, 'email');
LOG.debug(LogSection.eSYS, 'sending email message', { type: NotifyType[emailPackage.type], sendTo: emailPackage.sendTo?.join(',') }, 'Notify.testEmail');

const promise = NotifyEmail.sendMessage(emailPackage)
.then((status) => {
if (!status.success) {
LOG.error(LogSection.eSYS, status.message, status.data, 'Notify.testEmail');
errors.push(`${i}: ${status.message} - ${status.data?.error || 'unknown error'}`);
}
return status;
});

promises.push(promise);
}
await Promise.allSettled(promises);

// close our profiler and return results
const metrics = rateManager?.getMetrics();
const result = await LOG.profileEnd(profileKey);
return { success: (errors.length<=0), message: `finished testing ${numEmails} email messages.`, data: { message: result.message, maxRate: metrics?.rates.max, avgRate: metrics?.rates.average, errors: (errors.length>0) ? errors?.join(' | ') : null } };
}
//#endregion

//#region SLACK
//#endregio
public static configureSlack(env: ENVIRONMENT_TYPE, apiKey: string, targetRate?: number, burstRate?: number, burstThreshold?: number): NotifyResult {
return NotifySlack.configure(env,apiKey,targetRate,burstRate,burstThreshold);
}

// cast the returns to NotifyResult so it's consistent with what is exported
// NOTE: not exporting raw variant currently due to the specialized knowledge of blocks required for it to work
public static sendSlackMessage = NotifySlack.sendMessage as (params: NotifyPackage, channel?: SlackChannel) => Promise<NotifyResult>;
public static clearSlackChannel = NotifySlack.clearChannel as (channel?: SlackChannel) => Promise<NotifyResult>;

// testing slack messages
public static async testSlack(numMessages: number, channel?: SlackChannel): Promise<NotifyResult> {

const rateManager = NotifySlack.getRateManager();
const hasRateManager: boolean = rateManager ? true : false;
const config = rateManager?.getConfig();

// create our profiler
// we use a random string in case another test or profile is run to avoid collisisons
const profileKey: string = `SlackTest_${Math.random().toString(36).substring(2, 6)}`;
await LOG.profile(profileKey, LogSection.eSYS, `Slack test: ${new Date().toLocaleString()}`, {
numMessages,
rateManager: hasRateManager,
...(hasRateManager === true && config && {
config: (({ onPost: _onPost, ...rest }) => rest)(config) // Exclude onPost
})
},'Notify.testSlack');

// test our slack
const errors: string[] = [];
const promises: Promise<NotifyResult>[] = [];

for(let i=0; i<numMessages; ++i) {
const slackPackage: NotifyPackage = randomNotifyPackage(i, 'slack');
LOG.debug(LogSection.eSYS, 'sending slack message', { channel, type: NotifyType[slackPackage.type], sendTo: slackPackage.sendTo?.join(',') }, 'Notify.testSlack');

const promise = NotifySlack.sendMessage(slackPackage, channel)
.then((status) => {
if (!status.success) {
LOG.error(LogSection.eSYS, status.message, status.data, 'Notify.testSlack');
errors.push(`${i}: ${status.message} - ${status.data?.error || 'unknown error'}`);
}
return status;
});

promises.push(promise);
}
await Promise.allSettled(promises);

// close our profiler and return results
const metrics = rateManager?.getMetrics();
const result = await LOG.profileEnd(profileKey);
return { success: (errors.length<=0), message: `finished testing ${numMessages} slack messages.`, data: { message: result.message, maxRate: metrics?.rates.max, avgRate: metrics?.rates.average, errors: (errors.length>0) ? errors?.join(' | ') : null } };
}
//#endregion
}

// export shared types so they can be accessed via Notify
export { NotifyPackage, NotifyChannel, NotifyType };
export { NotifyPackage, NotifyUserGroup, NotifyType, SlackChannel };
Loading

0 comments on commit dfb1a68

Please sign in to comment.