diff --git a/jest.config.ts b/jest.config.ts index b1728a4..f9a5b4e 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -9,7 +9,7 @@ const jestConfig: JestConfigWithTsJest = { transform: { "^.+\\.(t|j)s$": "ts-jest", }, - collectCoverageFrom: ["**/*.ts", "!coverage/**", "!utils/noop.ts", "!index.ts", "!app.ts"], + collectCoverageFrom: ["**/*.ts", "!coverage/**", "!utils/noop.ts", "!index.ts", "!app.ts", "!**/models/*.ts"], coverageDirectory: "../coverage", testEnvironment: "node", roots: [""], diff --git a/src/app.ts b/src/app.ts index 4b2d758..b105105 100644 --- a/src/app.ts +++ b/src/app.ts @@ -5,25 +5,18 @@ import chalk from "chalk"; import prettyMilliseconds from "pretty-ms"; import pluralize from "pluralize"; -import { UserRepository } from "@repositories/user"; -import { UserLogRepository } from "@repositories/user-log"; +import { UserRepository, User } from "@repositories/user"; +import { UserLogRepository, UserLog } from "@repositories/user-log"; -import { User, UserData } from "@repositories/models/user"; -import { UserLog, UserLogType } from "@root/repositories/models/user-log"; +import { DEFAULT_TASKS, TaskData } from "@tasks"; -import { Config } from "@utils/config"; -import { throttle } from "@utils/throttle"; -import { mapBy } from "@utils/mapBy"; -import { measureTime } from "@utils/measureTime"; -import { sleep } from "@utils/sleep"; -import { Logger } from "@utils/logger"; -import { Loggable } from "@utils/types"; -import { getDiff } from "@utils/getDiff"; +import { Config, throttle, measureTime, sleep, Logger, Loggable } from "@utils"; export class App extends Loggable { private readonly followerDataSource: DataSource; private readonly userRepository: UserRepository; private readonly userLogRepository: UserLogRepository; + private readonly taskClasses = DEFAULT_TASKS; private cleaningUp = false; private config: Config | null = null; @@ -71,8 +64,7 @@ export class App extends Loggable { this.config = await Config.create(this.configFilePath); if (!this.config) { - process.exit(-1); - return; + throw new Error("config is not loaded."); } const watchers = this.config.watchers; @@ -154,114 +146,22 @@ export class App extends Loggable { throw new Error("Config is not loaded."); } - const startedDate = new Date(); - const allUserData: UserData[] = []; - for (const [, watcher] of this.config.watchers) { - try { - const userData = await watcher.doWatch(); - - allUserData.push(...userData); - } catch (e) { - if (!(e instanceof Error)) { - throw e; - } - - this.logger.error("an error occurred while watching through `{green}`: {}", [ - watcher.getName(), - e.message, - ]); - - process.exit(-1); - } - } - - this.logger.info(`all {} {} collected.`, [allUserData.length, pluralize("follower", allUserData.length)]); - - const followingMap = await this.userLogRepository.getFollowStatusMap(); - const oldUsers = await this.userRepository.find(); - const oldUsersMap = mapBy(oldUsers, "id"); - - const newUsers = await this.userRepository.createFromData(allUserData, startedDate); - const newUserMap = mapBy(newUsers, "id"); - - // find user renaming their displayName or userId - const displayNameRenamedUsers = getDiff(newUsers, oldUsers, "uniqueId", "displayName"); - const userIdRenamedUsers = getDiff(newUsers, oldUsers, "uniqueId", "userId"); - - // find users who are not followed yet. - const newFollowers = newUsers.filter(p => !followingMap[p.id]); - const unfollowers = oldUsers.filter(p => !newUserMap[p.id] && followingMap[p.id]); - - const renameLogs: UserLog[] = []; - if (displayNameRenamedUsers.length || userIdRenamedUsers.length) { - const displayName = displayNameRenamedUsers.map(p => { - const oldUser = oldUsersMap[p.id]; - const userLog = this.userLogRepository.create(); - userLog.type = UserLogType.RenameDisplayName; - userLog.user = p; - userLog.oldDisplayName = oldUser?.displayName; - userLog.oldUserId = oldUser?.userId; - - return userLog; - }); - - const userId = userIdRenamedUsers.map(p => { - const oldUser = oldUsersMap[p.id]; - const userLog = this.userLogRepository.create(); - userLog.type = UserLogType.RenameUserId; - userLog.user = p; - userLog.oldDisplayName = oldUser?.displayName; - userLog.oldUserId = oldUser?.userId; - - return userLog; - }); - - renameLogs.push(...displayName, ...userId); - } - - const newLogs = await this.userLogRepository.batchWriteLogs( - [ - [newFollowers, UserLogType.Follow], - [unfollowers, UserLogType.Unfollow], - ], - renameLogs, - ); - - this.logger.info(`all {} {} saved`, [newLogs.length, pluralize("log", newLogs.length)]); - - this.logger.info("tracked {} new {}", [newFollowers.length, pluralize("follower", newFollowers.length)]); - this.logger.info("tracked {} {}", [unfollowers.length, pluralize("unfollower", unfollowers.length)]); - - const notifiers = this.config.notifiers; - if (!notifiers.length) { - this.logger.info("no notifiers are configured. skipping notification..."); - return; - } - - const ignoredIds = this.config.ignores; - if (ignoredIds.length) { - this.logger.info("ignoring {}", [pluralize("user", ignoredIds.length, true)]); - - for (const log of newLogs) { - if (!ignoredIds.includes(log.user.id)) { - continue; - } + const taskData: TaskData[] = []; + for (const TaskClass of this.taskClasses) { + const taskInstance = new TaskClass( + this.config.watchers.map(([, watcher]) => watcher), + this.config.notifiers.map(([, notifier]) => notifier), + this.userRepository, + this.userLogRepository, + TaskClass.name, + ); - newLogs.splice(newLogs.indexOf(log), 1); + const data = await taskInstance.doWork(taskData); + if (data.type === "terminate") { + return; } - } - if (newLogs.length <= 0) { - return; - } - - const watcherMap = this.config.watcherMap; - for (const [, notifier] of notifiers) { - await notifier.notify( - newLogs.map(log => { - return [watcherMap[log.user.from], log]; - }), - ); + taskData.push(data); } }; } diff --git a/src/notifiers/base.ts b/src/notifiers/base.ts index 4093dcf..4eb3fe6 100644 --- a/src/notifiers/base.ts +++ b/src/notifiers/base.ts @@ -1,8 +1,11 @@ -import { NotifyPair } from "@notifiers/type"; +import { capitalCase } from "change-case"; + +import { UserLog, UserLogType } from "@repositories/models/user-log"; + +import { UserLogMap } from "@notifiers/type"; -import { Loggable } from "@utils/types"; -import { UserLogType } from "@repositories/models/user-log"; import { Logger } from "@utils/logger"; +import { Loggable } from "@utils/types"; export abstract class BaseNotifier extends Loggable { protected constructor(name: TType) { @@ -15,18 +18,17 @@ export abstract class BaseNotifier extends Loggable public abstract initialize(): Promise; - public abstract notify(logs: NotifyPair[]): Promise; + public abstract notify(logs: UserLog[], logMap: UserLogMap): Promise; - protected formatNotify(pair: NotifyPair): string { - const [watcher, log] = pair; + protected formatNotify(log: UserLog): string { const { user } = log; if (log.type === UserLogType.RenameUserId || log.type === UserLogType.RenameDisplayName) { const tokens = [ - watcher.getName(), + capitalCase(user.from), log.oldDisplayName || "", log.oldUserId || "", - watcher.getProfileUrl(log.user), + user.profileUrl, log.type === UserLogType.RenameDisplayName ? "" : "@", log.type === UserLogType.RenameUserId ? user.userId : user.displayName, ]; @@ -34,7 +36,12 @@ export abstract class BaseNotifier extends Loggable return Logger.format("[{}] [{} (@{})]({}) → {}{}", ...tokens); } - const tokens = [watcher.getName(), user.displayName, user.userId, watcher.getProfileUrl(user)]; - return Logger.format("[{}] [{} (@{})]({})", ...tokens); + return Logger.format( + "[{}] [{} (@{})]({})", + capitalCase(user.from), + user.displayName, + user.userId, + user.profileUrl, + ); } } diff --git a/src/notifiers/discord.ts b/src/notifiers/discord.ts index 109eb29..db8e650 100644 --- a/src/notifiers/discord.ts +++ b/src/notifiers/discord.ts @@ -1,10 +1,10 @@ import pluralize from "pluralize"; import dayjs from "dayjs"; -import { UserLogType } from "@repositories/models/user-log"; +import { UserLog, UserLogType } from "@repositories/models/user-log"; import { BaseNotifier } from "@notifiers/base"; -import { BaseNotifierOption, NotifyPair } from "@notifiers/type"; +import { BaseNotifierOption, UserLogMap } from "@notifiers/type"; import { Fetcher } from "@utils/fetcher"; import { Logger } from "@utils/logger"; @@ -40,12 +40,12 @@ export class DiscordNotifier extends BaseNotifier<"Discord"> { public async initialize() { this.webhookUrl = this.options.webhookUrl; } - public async notify(pairs: NotifyPair[]) { + public async notify(logs: UserLog[], logMap: UserLogMap) { if (!this.webhookUrl) { throw new Error("DiscordNotifier is not initialized"); } - if (pairs.length <= 0) { + if (logs.length <= 0) { return; } @@ -55,9 +55,9 @@ export class DiscordNotifier extends BaseNotifier<"Discord"> { { title: Logger.format( "Total {} {} {} found", - pairs.length, - pluralize("change", pairs.length), - pluralize("was", pairs.length), + logs.length, + pluralize("change", logs.length), + pluralize("was", logs.length), ), color: 5814783, fields: [], @@ -72,12 +72,9 @@ export class DiscordNotifier extends BaseNotifier<"Discord"> { }; const fields: DiscordWebhookData["embeds"][0]["fields"] = []; - const followerLogs = pairs.filter(([, l]) => l.type === UserLogType.Follow); - const unfollowerLogs = pairs.filter(([, l]) => l.type === UserLogType.Unfollow); - const renameLogs = pairs.filter( - ([, l]) => l.type === UserLogType.RenameUserId || l.type === UserLogType.RenameDisplayName, - ); - + const followerLogs = logMap[UserLogType.Follow]; + const unfollowerLogs = logMap[UserLogType.Unfollow]; + const renameLogs = logMap[UserLogType.Rename]; if (followerLogs.length > 0) { fields.push(this.composeLogs(followerLogs, "🎉 {} new {}", "follower")); } @@ -100,15 +97,12 @@ export class DiscordNotifier extends BaseNotifier<"Discord"> { } private composeLogs( - logs: NotifyPair[], + logs: UserLog[], messageFormat: string, word: string, ): DiscordWebhookData["embeds"][0]["fields"][0] { - const { name, value } = this.generateEmbedField( - logs, - Logger.format(messageFormat, logs.length, pluralize(word, logs.length)), - ); - + const message = Logger.format(messageFormat, logs.length, pluralize(word, logs.length)); + const { name, value } = this.generateEmbedField(logs, message); const valueLines = [value]; if (logs.length > 10) { valueLines.push(`_... and ${logs.length - 10} more_`); @@ -119,7 +113,7 @@ export class DiscordNotifier extends BaseNotifier<"Discord"> { value: valueLines.join("\n"), }; } - private generateEmbedField(logs: NotifyPair[], title: string) { + private generateEmbedField(logs: UserLog[], title: string) { return { name: title, value: logs.slice(0, 10).map(this.formatNotify).join("\n"), diff --git a/src/notifiers/slack.ts b/src/notifiers/slack.ts index 81ad62c..c10d7f8 100644 --- a/src/notifiers/slack.ts +++ b/src/notifiers/slack.ts @@ -1,10 +1,10 @@ import { IncomingWebhook, IncomingWebhookSendArguments } from "@slack/webhook"; import { BaseNotifier } from "@notifiers/base"; -import { BaseNotifierOption, NotifyPair } from "@notifiers/type"; +import { BaseNotifierOption, UserLogMap } from "@notifiers/type"; -import { groupNotifies } from "@utils/groupNotifies"; import { Logger } from "@utils/logger"; +import { UserLog } from "@repositories/models/user-log"; export interface SlackNotifierOptions extends BaseNotifierOption { webhookUrl: string; @@ -24,24 +24,16 @@ export class SlackNotifier extends BaseNotifier<"Slack"> { public async initialize(): Promise { return; } - public async notify(logs: NotifyPair[]): Promise { - const { follow, unfollow, rename } = groupNotifies(logs); - const targets: [NotifyPair[], number, string, string][] = [ + public async notify(logs: UserLog[], logMap: UserLogMap): Promise { + const { follow, unfollow, rename } = logMap; + const targets: [UserLog[], number, string, string][] = [ [follow.slice(0, MAX_ITEMS_PER_MESSAGE), follow.length, "🎉 {} new {}", "follower"], [unfollow.slice(0, MAX_ITEMS_PER_MESSAGE), unfollow.length, "❌ {} {}", "unfollower"], [rename.slice(0, MAX_ITEMS_PER_MESSAGE), rename.length, "✏️ {} {}", "rename"], ]; const result: IncomingWebhookSendArguments = { - blocks: [ - { - type: "section", - text: { - type: "mrkdwn", - text: "_*🦜 Cage Report*_", - }, - }, - ], + blocks: [{ type: "section", text: { type: "mrkdwn", text: "_*🦜 Cage Report*_" } }], }; let shouldNotify = false; @@ -78,7 +70,7 @@ export class SlackNotifier extends BaseNotifier<"Slack"> { } } - protected formatNotify(pair: NotifyPair): string { - return super.formatNotify(pair).replace(/ \[(.*? \(@.*?\))\]\((.*?)\)/g, "<$2|$1>"); + protected formatNotify(log: UserLog): string { + return super.formatNotify(log).replace(/ \[(.*? \(@.*?\))\]\((.*?)\)/g, "<$2|$1>"); } } diff --git a/src/notifiers/telegram/constants.ts b/src/notifiers/telegram/constants.ts index 8833580..e7dd5f6 100644 --- a/src/notifiers/telegram/constants.ts +++ b/src/notifiers/telegram/constants.ts @@ -1,19 +1,9 @@ -export const TELEGRAM_FOLLOWERS_TEMPLATE = ` -*🎉 {} {}* +import { UserLogType } from "@repositories/models/user-log"; -{} -{} -`.trim(); -export const TELEGRAM_UNFOLLOWERS_TEMPLATE = ` -*❌ {} {}* +export const CONTENT_TEMPLATES: Partial> = { + [UserLogType.Follow]: ["**🎉 {}**\n\n{}", "new follower"], + [UserLogType.Unfollow]: ["**❌ {}**\n\n{}", "unfollower"], + [UserLogType.Rename]: ["**✏️ {}**\n\n{}", "rename"], +}; -{} -{} -`.trim(); -export const TELEGRAM_RENAMES_TEMPLATE = ` -*✏️ {} {}* - -{} -{} -`.trim(); -export const TELEGRAM_LOG_COUNT = 25; +export const MAXIMUM_LOG_COUNT = 50; diff --git a/src/notifiers/telegram/index.ts b/src/notifiers/telegram/index.ts index 0037382..2d92f14 100644 --- a/src/notifiers/telegram/index.ts +++ b/src/notifiers/telegram/index.ts @@ -1,15 +1,15 @@ import pluralize from "pluralize"; import { BaseNotifier } from "@notifiers/base"; -import { BaseNotifierOption, NotifyPair } from "@notifiers/type"; -import { TELEGRAM_LOG_COUNT } from "@notifiers/telegram/constants"; +import { BaseNotifierOption, UserLogMap } from "@notifiers/type"; import { NotifyResponse, TelegramNotificationData, TokenResponse } from "@notifiers/telegram/types"; +import { CONTENT_TEMPLATES, MAXIMUM_LOG_COUNT } from "@notifiers/telegram/constants"; + +import { UserLog, UserLogType } from "@repositories/models/user-log"; import { Fetcher } from "@utils/fetcher"; -import { groupNotifies } from "@utils/groupNotifies"; -import { Logger } from "@utils/logger"; import { HttpError } from "@utils/httpError"; -import { generateNotificationTargets } from "@notifiers/telegram/utils"; +import { Logger } from "@utils/logger"; export interface TelegramNotifierOptions extends BaseNotifierOption { token: string; @@ -27,31 +27,40 @@ export class TelegramNotifier extends BaseNotifier<"Telegram"> { public async initialize(): Promise { this.currentToken = await this.acquireToken(); } - public async notify(logs: NotifyPair[]): Promise { - const { follow, unfollow, rename } = groupNotifies(logs); - const targets = generateNotificationTargets({ - unfollowers: unfollow, - followers: follow, - renames: rename, - }); - - const notifyData: Partial = {}; - for (const { fieldName, countFieldName, count, word, template, pairs } of targets) { - if (count <= 0) { + public async notify(_: UserLog[], logMap: UserLogMap): Promise { + const reportTargets: UserLogType[] = [UserLogType.Follow, UserLogType.Unfollow, UserLogType.Rename]; + const content: Partial> = {}; + for (const type of reportTargets) { + const logs = logMap[type]; + if (logs.length <= 0) { continue; } - notifyData[countFieldName] = count; - notifyData[fieldName] = Logger.format( - template, - count, - pluralize(word, count), - pairs.map(this.formatNotify).join("\n"), - count > TELEGRAM_LOG_COUNT ? `_... and ${count - TELEGRAM_LOG_COUNT} more_` : "", - ).trim(); + const template = CONTENT_TEMPLATES[type]; + if (!template) { + throw new Error(`There is no message template for log type '${type}'`); + } + + const [title, action] = template; + //TODO: replace this to adjusting the length of the message by content not hard-coded + let messageContent = logs.slice(0, MAXIMUM_LOG_COUNT).map(this.formatNotify).join("\n"); + if (logs.length > MAXIMUM_LOG_COUNT) { + const remainCount = logs.length - MAXIMUM_LOG_COUNT; + messageContent += `\n\n_... and ${remainCount} more_`; + } + + const text = Logger.format(title, pluralize(action, logs.length, true), messageContent); + content[type] = [text, logs.length]; } - await this.pushNotify(notifyData); + await this.pushNotify({ + followers: content[UserLogType.Follow]?.[0], + unfollowers: content[UserLogType.Unfollow]?.[0], + renames: content[UserLogType.Rename]?.[0], + followerCount: content[UserLogType.Follow]?.[1], + unfollowerCount: content[UserLogType.Unfollow]?.[1], + renameCount: content[UserLogType.Rename]?.[1], + }); } private getEndpoint(path: string) { diff --git a/src/notifiers/telegram/types.ts b/src/notifiers/telegram/types.ts index b3f3db3..0780a33 100644 --- a/src/notifiers/telegram/types.ts +++ b/src/notifiers/telegram/types.ts @@ -1,6 +1,3 @@ -import { SelectOnly } from "@utils/types"; -import { NotifyPair } from "@notifiers/type"; - export interface TelegramNotificationData { followers?: string; unfollowers?: string; @@ -17,12 +14,3 @@ export interface TokenResponse { export interface NotifyResponse { success: boolean; } - -export interface NotificationTarget { - fieldName: keyof SelectOnly; - countFieldName: keyof SelectOnly; - word: string; - template: string; - count: number; - pairs: NotifyPair[]; -} diff --git a/src/notifiers/telegram/utils.ts b/src/notifiers/telegram/utils.ts deleted file mode 100644 index cf461ee..0000000 --- a/src/notifiers/telegram/utils.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { SelectOnly } from "@utils/types"; -import { NotificationTarget, TelegramNotificationData } from "@notifiers/telegram/types"; -import { NotifyPair } from "@notifiers/type"; -import { - TELEGRAM_FOLLOWERS_TEMPLATE, - TELEGRAM_LOG_COUNT, - TELEGRAM_RENAMES_TEMPLATE, - TELEGRAM_UNFOLLOWERS_TEMPLATE, -} from "@notifiers/telegram/constants"; - -export function generateNotificationTargets( - pairs: Record, NotifyPair[]>, -): NotificationTarget[] { - return [ - { - fieldName: "followers", - countFieldName: "followerCount", - word: "new follower", - template: TELEGRAM_FOLLOWERS_TEMPLATE, - pairs: pairs.followers.slice(0, TELEGRAM_LOG_COUNT), - count: pairs.followers.length, - }, - { - fieldName: "unfollowers", - countFieldName: "unfollowerCount", - word: "unfollower", - template: TELEGRAM_UNFOLLOWERS_TEMPLATE, - pairs: pairs.unfollowers.slice(0, TELEGRAM_LOG_COUNT), - count: pairs.unfollowers.length, - }, - { - fieldName: "renames", - countFieldName: "renameCount", - word: "rename", - template: TELEGRAM_RENAMES_TEMPLATE, - pairs: pairs.renames.slice(0, TELEGRAM_LOG_COUNT), - count: pairs.renames.length, - }, - ]; -} diff --git a/src/notifiers/type.ts b/src/notifiers/type.ts index 303b137..0321b34 100644 --- a/src/notifiers/type.ts +++ b/src/notifiers/type.ts @@ -1,10 +1,8 @@ -import { BaseWatcher } from "@watchers/base"; import { BaseNotifier } from "@notifiers/base"; - -import { UserLog } from "@repositories/models/user-log"; +import { UserLog, UserLogType } from "@repositories/models/user-log"; export interface BaseNotifierOption { type: TNotifier extends BaseNotifier ? Lowercase : string; } -export type NotifyPair = [BaseWatcher, UserLog]; +export type UserLogMap = Record; diff --git a/src/repositories/base.ts b/src/repositories/base.ts index dbd92f2..0138344 100644 --- a/src/repositories/base.ts +++ b/src/repositories/base.ts @@ -1,53 +1,7 @@ import { BaseEntity, Repository } from "typeorm"; -import { AsyncFn } from "@utils/types"; - export abstract class BaseRepository extends Repository { - private readonly buffer: TEntity[] = []; - private inTransaction = false; - protected constructor(repository: Repository) { super(repository.target, repository.manager, repository.queryRunner); } - - public async transaction(task: AsyncFn) { - this.inTransaction = true; - - try { - await task(); - } catch (error) { - await this.closeTransaction(); - throw error; - } - - await this.closeTransaction(); - } - - private async closeTransaction() { - if (this.buffer.length === 0) { - return; - } - - const items = [...this.buffer]; - this.buffer.length = 0; - this.inTransaction = false; - - await this.save(items); - } - - protected async saveItems(item: TEntity): Promise; - protected async saveItems(items: TEntity[]): Promise; - protected async saveItems(items: TEntity[] | TEntity): Promise { - if (this.inTransaction) { - this.buffer.push(...(Array.isArray(items) ? items : [items])); - return items; - } - - if (!Array.isArray(items)) { - const [result] = await this.save([items]); - return result; - } - - return this.save(items); - } } diff --git a/src/repositories/models/user.ts b/src/repositories/models/user.ts index 0f8a19a..851d2cf 100644 --- a/src/repositories/models/user.ts +++ b/src/repositories/models/user.ts @@ -2,8 +2,6 @@ import { BaseEntity, Column, CreateDateColumn, Entity, OneToMany, RelationId, Up import { UserLog } from "@root/repositories/models/user-log"; -export type UserData = Pick; - @Entity({ name: "users" }) export class User extends BaseEntity { @Column({ type: "varchar", length: 255, primary: true }) @@ -24,6 +22,9 @@ export class User extends BaseEntity { @Column({ type: "datetime" }) public lastlyCheckedAt!: Date; + @Column({ type: "varchar" }) + public profileUrl!: string; + @CreateDateColumn() public createdAt!: Date; diff --git a/src/repositories/user-log.ts b/src/repositories/user-log.ts index f500f82..432050f 100644 --- a/src/repositories/user-log.ts +++ b/src/repositories/user-log.ts @@ -1,8 +1,6 @@ import { Repository } from "typeorm"; import { BaseRepository } from "@repositories/base"; - -import { User } from "@repositories/models/user"; import { UserLog, UserLogType } from "@repositories/models/user-log"; import { mapBy } from "@utils/mapBy"; @@ -12,42 +10,6 @@ export class UserLogRepository extends BaseRepository { super(repository); } - public async writeLog(user: User, type: UserLog["type"]) { - const userLog = this.create(); - userLog.user = user; - userLog.type = type; - - return this.saveItems(userLog); - } - public async writeLogs(users: User[], type: UserLog["type"]) { - const userLogs = users.map(user => { - const userLog = this.create(); - userLog.user = user; - userLog.type = type; - - return userLog; - }); - - return this.saveItems(userLogs); - } - public async batchWriteLogs(items: Array<[User[], UserLog["type"]]>, rawLogs?: UserLog[]) { - const userLogs = items.flatMap(([users, type]) => - users.map(user => { - const userLog = this.create(); - userLog.user = user; - userLog.type = type; - - return userLog; - }), - ); - - if (rawLogs) { - userLogs.push(...rawLogs); - } - - return this.saveItems(userLogs); - } - public async getFollowStatusMap() { const data = await this.createQueryBuilder() .select("type") @@ -60,3 +22,5 @@ export class UserLogRepository extends BaseRepository { return mapBy(data, "userId", item => item.type === UserLogType.Follow); } } + +export * from "./models/user-log"; diff --git a/src/repositories/user.ts b/src/repositories/user.ts index fca363b..faf37f7 100644 --- a/src/repositories/user.ts +++ b/src/repositories/user.ts @@ -1,30 +1,12 @@ import { Repository } from "typeorm"; import { BaseRepository } from "@repositories/base"; -import { User, UserData } from "@repositories/models/user"; +import { User } from "@repositories/models/user"; export class UserRepository extends BaseRepository { public constructor(repository: Repository) { super(repository); } - - public async createFromData(data: UserData | UserData[], lastlyCheckedAt: Date = new Date()) { - if (!Array.isArray(data)) { - data = [data]; - } - - const users = data.map(item => { - const user = this.create(); - user.id = `${item.from}:${item.uniqueId}`; - user.displayName = item.displayName; - user.userId = item.userId; - user.uniqueId = item.uniqueId; - user.from = item.from; - user.lastlyCheckedAt = lastlyCheckedAt; - - return user; - }); - - return this.saveItems(users); - } } + +export * from "./models/user"; diff --git a/src/tasks/base.spec.ts b/src/tasks/base.spec.ts new file mode 100644 index 0000000..998d9ba --- /dev/null +++ b/src/tasks/base.spec.ts @@ -0,0 +1,52 @@ +import stripAnsi from "strip-ansi"; + +import { BaseTask, TaskData } from "@tasks/base"; +import { UserLog } from "@repositories/models/user-log"; +import { WorkOptions } from "@utils"; + +describe("BaseTask", () => { + it("should determine given task data type", function () { + const data: TaskData[] = [ + { type: "new-users", data: [] }, + { type: "new-logs", data: [] }, + { type: "notify" }, + { type: "save", savedCount: 0 }, + { type: "terminate", reason: "" }, + { type: "skip", reason: "" }, + ]; + + expect(data.map(BaseTask.isNewUsersData)).toEqual([true, false, false, false, false, false]); + expect(data.map(BaseTask.isNewLogsData)).toEqual([false, true, false, false, false, false]); + expect(data.map(BaseTask.isTerminateData)).toEqual([false, false, false, false, true, false]); + expect(data.map(BaseTask.isSkipData)).toEqual([false, false, false, false, false, true]); + }); + + it("should process task with rich logging", async () => { + const taskDataArray: [TaskData, string][] = [ + [{ type: "new-logs", data: [{} as UserLog] }, "Created 1 new logs"], + [{ type: "skip", reason: "mock-reason" }, "Skipping `Mock` task: mock-reason"], + [{ type: "terminate", reason: "mock-reason" }, "Terminating whole task pipeline: mock-reason"], + ]; + + for (const [item, targetMessage] of taskDataArray) { + class MockTask extends BaseTask { + public async process(): Promise { + return item; + } + } + + const task = new MockTask([], [], null as any, null as any, "MockTask"); + let logMessage = ""; + Object.defineProperty(task, "logger", { + value: { + work: jest.fn().mockImplementation((options: WorkOptions) => options.work()), + info: jest.fn().mockImplementation((message: string) => (logMessage = message)), + }, + }); + + await task.doWork([]); + expect(task["logger"].info).toBeCalledTimes(1); + expect(stripAnsi(logMessage)).toBe(targetMessage); + } + }); +}); diff --git a/src/tasks/base.ts b/src/tasks/base.ts new file mode 100644 index 0000000..6fccc48 --- /dev/null +++ b/src/tasks/base.ts @@ -0,0 +1,101 @@ +import { BaseNotifier } from "@notifiers/base"; +import { BaseWatcher } from "@watchers/base"; + +import { UserRepository } from "@repositories/user"; +import { UserLogRepository } from "@repositories/user-log"; +import { User } from "@repositories/models/user"; +import { UserLog } from "@repositories/models/user-log"; + +import { Loggable, Logger } from "@utils"; + +interface NewUsersTaskData { + type: "new-users"; + data: User[]; +} +interface NewLogsTaskData { + type: "new-logs"; + data: UserLog[]; +} +interface NotifyTaskData { + type: "notify"; +} +interface SaveTaskData { + type: "save"; + savedCount: number; +} +interface TerminateTaskData { + type: "terminate"; + reason: string; +} +interface SkipTaskData { + type: "skip"; + reason: string; +} + +export type TaskData = + | NewUsersTaskData + | NewLogsTaskData + | NotifyTaskData + | SaveTaskData + | TerminateTaskData + | SkipTaskData; +export type TaskClass = new (...args: ConstructorParameters) => BaseTask; + +export abstract class BaseTask extends Loggable { + public constructor( + protected readonly watchers: ReadonlyArray>, + protected readonly notifiers: ReadonlyArray>, + protected readonly userRepository: UserRepository, + protected readonly userLogRepository: UserLogRepository, + name: string, + ) { + super(name.replace(/task$/i, "")); + } + + public static isNewUsersData(data: TaskData): data is NewUsersTaskData { + return data.type === "new-users"; + } + public static isNewLogsData(data: TaskData): data is NewLogsTaskData { + return data.type === "new-logs"; + } + public static isTerminateData(data: TaskData): data is TerminateTaskData { + return data.type === "terminate"; + } + public static isSkipData(data: TaskData): data is SkipTaskData { + return data.type === "skip"; + } + + public async doWork(previousData: TaskData[]): Promise { + const data = await this.logger.work({ + message: Logger.format("processing `{green}` task", this.getName()), + level: "info", + work: () => this.process(previousData), + }); + + this.logData(data); + return data; + } + + protected abstract process(previousData: TaskData[]): Promise; + + protected terminate(reason: string): TerminateTaskData { + return { type: "terminate", reason }; + } + protected skip(reason: string): SkipTaskData { + return { type: "skip", reason }; + } + + private logData(data: TaskData) { + if (BaseTask.isNewLogsData(data) && data.data.length > 0) { + this.logger.info(Logger.format("Created {green} new logs", data.data.length)); + } + + if (BaseTask.isTerminateData(data)) { + this.logger.info(Logger.format("Terminating whole task pipeline: {red}", data.reason)); + } + + if (BaseTask.isSkipData(data)) { + this.logger.info(Logger.format("Skipping `{green}` task: {red}", this.getName(), data.reason)); + } + } +} diff --git a/src/tasks/grab-user.spec.ts b/src/tasks/grab-user.spec.ts new file mode 100644 index 0000000..97461d0 --- /dev/null +++ b/src/tasks/grab-user.spec.ts @@ -0,0 +1,49 @@ +import { GrabUserTask } from "@tasks/grab-user"; +import { BaseWatcher } from "@watchers/base"; + +describe("GrabUserTask", () => { + it("should grab users from watchers", async function () { + const watchers = [ + { + doWatch: jest.fn().mockResolvedValueOnce([{ id: 1, name: "user1" }]), + }, + { + doWatch: jest.fn().mockResolvedValueOnce([{ id: 2, name: "user2" }]), + }, + ] as any as BaseWatcher[]; + + const task = new GrabUserTask(watchers, [], null as any, null as any, GrabUserTask.name); + const result = await task.process(); + + expect(result).toEqual({ + type: "new-users", + data: [ + { id: 1, name: "user1" }, + { id: 2, name: "user2" }, + ], + }); + expect(watchers[0].doWatch).toBeCalledTimes(1); + expect(watchers[1].doWatch).toBeCalledTimes(1); + }); + + it("should terminate if no users found", async function () { + const watchers = [ + { + doWatch: jest.fn().mockResolvedValueOnce([]), + }, + { + doWatch: jest.fn().mockResolvedValueOnce([]), + }, + ] as any as BaseWatcher[]; + + const task = new GrabUserTask(watchers, [], null as any, null as any, GrabUserTask.name); + const result = await task.process(); + + expect(result).toEqual({ + type: "terminate", + reason: "No followers found", + }); + expect(watchers[0].doWatch).toBeCalledTimes(1); + expect(watchers[1].doWatch).toBeCalledTimes(1); + }); +}); diff --git a/src/tasks/grab-user.ts b/src/tasks/grab-user.ts new file mode 100644 index 0000000..5c4936f --- /dev/null +++ b/src/tasks/grab-user.ts @@ -0,0 +1,23 @@ +import { User } from "@repositories/models/user"; + +import { BaseTask, TaskData } from "@tasks/base"; + +export class GrabUserTask extends BaseTask { + public async process(): Promise { + const taskStartedAt = new Date(); + const allUsers: User[] = []; + for (const watcher of this.watchers) { + const users = await watcher.doWatch(taskStartedAt); + allUsers.push(...users); + } + + if (allUsers.length <= 0) { + return this.terminate("No followers found"); + } + + return { + type: "new-users", + data: allUsers, + }; + } +} diff --git a/src/tasks/index.spec.ts b/src/tasks/index.spec.ts new file mode 100644 index 0000000..cd2ea48 --- /dev/null +++ b/src/tasks/index.spec.ts @@ -0,0 +1,8 @@ +import { DEFAULT_TASKS } from "@tasks/index"; + +describe("Tasks", () => { + it("should provide predefined task array", () => { + expect(DEFAULT_TASKS).toBeDefined(); + expect(DEFAULT_TASKS.length).toBeGreaterThan(0); + }); +}); diff --git a/src/tasks/index.ts b/src/tasks/index.ts new file mode 100644 index 0000000..322f1fd --- /dev/null +++ b/src/tasks/index.ts @@ -0,0 +1,22 @@ +import { TaskClass } from "@tasks/base"; +import { NotifyTask } from "@tasks/notify"; +import { UnfollowerTask } from "@tasks/unfollower"; +import { NewFollowerTask } from "@tasks/new-follower"; +import { GrabUserTask } from "@tasks/grab-user"; +import { SaveTask } from "@tasks/save"; +import { RenameTask } from "@tasks/rename"; + +export const DEFAULT_TASKS: ReadonlyArray = [ + GrabUserTask, + NewFollowerTask, + UnfollowerTask, + RenameTask, + NotifyTask, + SaveTask, +]; + +export * from "./base"; +export * from "./grab-user"; +export * from "./new-follower"; +export * from "./notify"; +export * from "./unfollower"; diff --git a/src/tasks/new-follower.spec.ts b/src/tasks/new-follower.spec.ts new file mode 100644 index 0000000..f641232 --- /dev/null +++ b/src/tasks/new-follower.spec.ts @@ -0,0 +1,29 @@ +import { NewFollowerTask } from "@tasks/new-follower"; +import { UserLogRepository, UserLogType } from "@repositories/user-log"; + +describe("NewFollowerTask", () => { + it("should detect new followers between old and new user items", async function () { + const mockUserLogRepository = { + getFollowStatusMap: jest.fn().mockResolvedValue({ + user1: true, + user2: false, + }), + + create: jest.fn().mockReturnValue({}), + } as unknown as UserLogRepository; + + const followers = [{ id: "user1" }, { id: "user2" }]; + const task = new NewFollowerTask([], [], null as any, mockUserLogRepository, NewFollowerTask.name); + const result = await task.process([{ type: "new-users", data: followers as any }]); + + expect(result).toEqual({ + type: "new-logs", + data: [ + { + type: UserLogType.Follow, + user: { id: "user2" }, + }, + ], + }); + }); +}); diff --git a/src/tasks/new-follower.ts b/src/tasks/new-follower.ts new file mode 100644 index 0000000..c44fbba --- /dev/null +++ b/src/tasks/new-follower.ts @@ -0,0 +1,21 @@ +import { BaseTask, TaskData } from "@tasks/base"; +import { UserLogType } from "@repositories/models/user-log"; + +export class NewFollowerTask extends BaseTask { + public async process(previousData: TaskData[]): Promise { + const followingMap = await this.userLogRepository.getFollowStatusMap(); + const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); + const newFollowers = newUsers.filter(p => !followingMap[p.id]); + + return { + type: "new-logs", + data: newFollowers.map(item => { + const log = this.userLogRepository.create(); + log.type = UserLogType.Follow; + log.user = item; + + return log; + }), + }; + } +} diff --git a/src/tasks/notify.spec.ts b/src/tasks/notify.spec.ts new file mode 100644 index 0000000..984ed00 --- /dev/null +++ b/src/tasks/notify.spec.ts @@ -0,0 +1,43 @@ +import { NotifyTask } from "@tasks/notify"; +import { BaseNotifier } from "@notifiers/base"; +import { UserLogType } from "@repositories/models/user-log"; + +describe("NotifyTask", () => { + it("should notify new logs through notifiers", async () => { + const mockNotifier = { notify: jest.fn() } as unknown as BaseNotifier; + const mockUserLogs = [ + { type: UserLogType.Follow } as any, + { type: UserLogType.Unfollow } as any, + { type: UserLogType.RenameUserId } as any, + { type: UserLogType.RenameDisplayName } as any, + ]; + + const task = new NotifyTask([], [mockNotifier], null as any, null as any, NotifyTask.name); + const result = await task.process([ + { + type: "new-logs", + data: mockUserLogs, + }, + ]); + + expect(result).toEqual({ type: "notify" }); + expect(mockNotifier.notify).toHaveBeenCalledTimes(1); + expect(mockNotifier.notify).toHaveBeenCalledWith(mockUserLogs, { + [UserLogType.Follow]: [mockUserLogs[0]], + [UserLogType.Unfollow]: [mockUserLogs[1]], + [UserLogType.RenameUserId]: [mockUserLogs[2]], + [UserLogType.RenameDisplayName]: [mockUserLogs[3]], + [UserLogType.Rename]: [mockUserLogs[3], mockUserLogs[2]], + }); + }); + + it("should skip notifying if there is no new logs", async () => { + const mockNotifier = { notify: jest.fn() } as unknown as BaseNotifier; + + const task = new NotifyTask([], [mockNotifier], null as any, null as any, NotifyTask.name); + const result = await task.process([]); + + expect(result).toEqual({ type: "skip", reason: "No new logs to notify was found" }); + expect(mockNotifier.notify).not.toHaveBeenCalled(); + }); +}); diff --git a/src/tasks/notify.ts b/src/tasks/notify.ts new file mode 100644 index 0000000..01f63dd --- /dev/null +++ b/src/tasks/notify.ts @@ -0,0 +1,21 @@ +import { BaseTask, TaskData } from "@tasks/base"; + +import { groupNotifies } from "@utils/groupNotifies"; + +export class NotifyTask extends BaseTask { + public async process(previousData: TaskData[]): Promise { + const newLogs = previousData.filter(BaseTask.isNewLogsData).flatMap(item => item.data); + if (newLogs.length <= 0) { + return this.skip("No new logs to notify was found"); + } + + const logMap = groupNotifies(newLogs); + for (const notifier of this.notifiers) { + await notifier.notify(newLogs, logMap); + } + + return { + type: "notify", + }; + } +} diff --git a/src/tasks/rename.spec.ts b/src/tasks/rename.spec.ts new file mode 100644 index 0000000..28b97b4 --- /dev/null +++ b/src/tasks/rename.spec.ts @@ -0,0 +1,78 @@ +import { BaseEntity } from "typeorm"; +import { User, UserRepository } from "@root/repositories/user"; +import { RenameTask } from "@tasks/rename"; +import { UserLogRepository, UserLogType } from "@repositories/user-log"; + +let mockUserId = 0; +function createMockUser(userId: string, displayName: string): Omit { + const id = `user${++mockUserId}`; + + return { + id, + from: "mock", + uniqueId: id, + userId, + displayName, + lastlyCheckedAt: new Date(), + profileUrl: "https://example.com/user1", + createdAt: new Date(), + updatedAt: new Date(), + userLogs: [], + userLogIds: [], + }; +} + +describe("RenameTask", () => { + it("should detect renamed users between old and new user items", async () => { + const mockNewUsers: Omit[] = [ + createMockUser("mock1", "mock1"), + createMockUser("mock2", "mock2"), + ]; + const mockOldUsers: Omit[] = [ + { ...mockNewUsers[0], displayName: "old1" }, + { ...mockNewUsers[1], userId: "old2" }, + ]; + + const task = new RenameTask( + [], + [], + { find: jest.fn().mockResolvedValue(mockOldUsers) } as unknown as UserRepository, + { create: jest.fn().mockImplementation(item => item || {}) } as unknown as UserLogRepository, + RenameTask.name, + ); + const result = await task.process([{ type: "new-users", data: mockNewUsers as unknown as User[] }]); + + expect(result).toMatchObject({ + type: "new-logs", + data: [ + { + type: UserLogType.RenameDisplayName, + user: { id: mockNewUsers[0].id, displayName: "mock1" }, + oldDisplayName: "old1", + }, + { + type: UserLogType.RenameUserId, + user: { id: mockNewUsers[1].id, userId: "mock2" }, + oldUserId: "old2", + }, + ], + }); + }); + + it("should not detect renamed users on new user items", async () => { + const mockNewUsers: Omit[] = [createMockUser("mock1", "mock1")]; + const task = new RenameTask( + [], + [], + { find: jest.fn().mockResolvedValue(mockNewUsers) } as unknown as UserRepository, + { create: jest.fn().mockImplementation(item => item || {}) } as unknown as UserLogRepository, + RenameTask.name, + ); + const result = await task.process([{ type: "new-users", data: [] }]); + + expect(result).toMatchObject({ + type: "new-logs", + data: [], + }); + }); +}); diff --git a/src/tasks/rename.ts b/src/tasks/rename.ts new file mode 100644 index 0000000..1f6686e --- /dev/null +++ b/src/tasks/rename.ts @@ -0,0 +1,41 @@ +import { BaseTask, TaskData } from "@tasks/base"; + +import { UserLogType } from "@repositories/models/user-log"; +import { User } from "@repositories/models/user"; + +import { getDiff, isRequired, mapBy } from "@utils"; + +export class RenameTask extends BaseTask { + private createGenerator( + logType: UserLogType.RenameUserId | UserLogType.RenameDisplayName, + oldUsersMap: Record, + ) { + return (item: User) => { + const oldUser = oldUsersMap[item.id]; + return this.userLogRepository.create({ + type: logType, + user: item, + oldDisplayName: oldUser.displayName, + oldUserId: oldUser.userId, + }); + }; + } + + public async process(previousData: TaskData[]): Promise { + const oldUsers = await this.userRepository.find(); + const oldUsersMap = mapBy(oldUsers, "id"); + const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); + + // find user renaming their displayName or userId + const displayNameRenamedUsers = getDiff(newUsers, oldUsers, "uniqueId", "displayName"); + const userIdRenamedUsers = getDiff(newUsers, oldUsers, "uniqueId", "userId"); + + return { + type: "new-logs", + data: [ + ...displayNameRenamedUsers.map(this.createGenerator(UserLogType.RenameDisplayName, oldUsersMap)), + ...userIdRenamedUsers.map(this.createGenerator(UserLogType.RenameUserId, oldUsersMap)), + ].filter(isRequired), + }; + } +} diff --git a/src/tasks/save.spec.ts b/src/tasks/save.spec.ts new file mode 100644 index 0000000..1fb2621 --- /dev/null +++ b/src/tasks/save.spec.ts @@ -0,0 +1,38 @@ +import { UserRepository } from "@repositories/user"; +import { UserLogRepository } from "@repositories/user-log"; + +import { SaveTask } from "@tasks/save"; + +describe("SaveTask", () => { + it("should save new users and user logs", async () => { + const mockUserRepository = { save: jest.fn() } as unknown as UserRepository; + const mockUserLogRepository = { save: jest.fn() } as unknown as UserLogRepository; + + const mockUsers = [{ id: "user1" }, { id: "user2" }] as any; + const mockUserLogs = [{ id: "log1" }, { id: "log2" }] as any; + const task = new SaveTask([], [], mockUserRepository, mockUserLogRepository, SaveTask.name); + + const result = await task.process([ + { type: "new-users", data: mockUsers }, + { type: "new-logs", data: mockUserLogs }, + ]); + + expect(result).toEqual({ type: "save", savedCount: 4 }); + expect(mockUserRepository.save).toHaveBeenCalledTimes(1); + expect(mockUserRepository.save).toHaveBeenCalledWith(mockUsers); + expect(mockUserLogRepository.save).toHaveBeenCalledTimes(1); + expect(mockUserLogRepository.save).toHaveBeenCalledWith(mockUserLogs); + }); + + it("should skip saving if there is no new users and user logs", async () => { + const mockUserRepository = { save: jest.fn() } as unknown as UserRepository; + const mockUserLogRepository = { save: jest.fn() } as unknown as UserLogRepository; + + const task = new SaveTask([], [], mockUserRepository, mockUserLogRepository, SaveTask.name); + const result = await task.process([]); + + expect(result).toEqual({ type: "skip", reason: "No new logs or users to save was found" }); + expect(mockUserRepository.save).not.toHaveBeenCalled(); + expect(mockUserLogRepository.save).not.toHaveBeenCalled(); + }); +}); diff --git a/src/tasks/save.ts b/src/tasks/save.ts new file mode 100644 index 0000000..2642885 --- /dev/null +++ b/src/tasks/save.ts @@ -0,0 +1,19 @@ +import { BaseTask, TaskData } from "@tasks/base"; + +export class SaveTask extends BaseTask { + public async process(previousData: TaskData[]): Promise { + const newLogs = previousData.filter(BaseTask.isNewLogsData).flatMap(item => item.data); + const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); + if (newLogs.length <= 0 && newUsers.length <= 0) { + return this.skip("No new logs or users to save was found"); + } + + await this.userRepository.save(newUsers); + await this.userLogRepository.save(newLogs); + + return { + type: "save", + savedCount: newLogs.length + newUsers.length, + }; + } +} diff --git a/src/tasks/unfollower.spec.ts b/src/tasks/unfollower.spec.ts new file mode 100644 index 0000000..773fd62 --- /dev/null +++ b/src/tasks/unfollower.spec.ts @@ -0,0 +1,27 @@ +import { UserLogRepository, UserLogType } from "@repositories/user-log"; +import { UserRepository } from "@repositories/user"; + +import { UnfollowerTask } from "@tasks/unfollower"; + +describe("UnfollowerTask", () => { + it("should detect unfollowers between old and new user items", async () => { + const mockUserRepository = { + find: jest.fn().mockResolvedValue([{ id: "user1" }, { id: "user2" }]), + } as unknown as UserRepository; + const mockUserLogRepository = { + getFollowStatusMap: jest.fn().mockResolvedValue({ user1: true, user2: true }), + create: jest.fn().mockImplementation(item => item || {}), + } as unknown as UserLogRepository; + + const task = new UnfollowerTask([], [], mockUserRepository, mockUserLogRepository, UnfollowerTask.name); + const result = await task.process([{ type: "new-users", data: [] }]); + + expect(result).toEqual({ + type: "new-logs", + data: [ + { type: UserLogType.Unfollow, user: { id: "user1" } }, + { type: UserLogType.Unfollow, user: { id: "user2" } }, + ], + }); + }); +}); diff --git a/src/tasks/unfollower.ts b/src/tasks/unfollower.ts new file mode 100644 index 0000000..303e695 --- /dev/null +++ b/src/tasks/unfollower.ts @@ -0,0 +1,23 @@ +import { BaseTask, TaskData } from "@tasks/base"; +import { mapBy } from "@utils/mapBy"; +import { UserLogType } from "@repositories/models/user-log"; + +export class UnfollowerTask extends BaseTask { + public async process(previousData: TaskData[]): Promise { + const followingMap = await this.userLogRepository.getFollowStatusMap(); + const oldUsers = await this.userRepository.find(); + const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); + const newUserMap = mapBy(newUsers, "id"); + const unfollowers = oldUsers.filter(p => !newUserMap[p.id] && followingMap[p.id]); + + return { + type: "new-logs", + data: unfollowers.map(item => + this.userLogRepository.create({ + type: UserLogType.Unfollow, + user: item, + }), + ), + }; + } +} diff --git a/src/utils/generateRandomString.ts b/src/utils/generateRandomString.ts deleted file mode 100644 index 96cd0cf..0000000 --- a/src/utils/generateRandomString.ts +++ /dev/null @@ -1,8 +0,0 @@ -export function generateRandomString(length: number, chars: string) { - let result = ""; - for (let i = length; i > 0; --i) { - result += chars[Math.floor(Math.random() * chars.length)]; - } - - return result; -} diff --git a/src/utils/getTimestamp.ts b/src/utils/getTimestamp.ts deleted file mode 100644 index b51b0ed..0000000 --- a/src/utils/getTimestamp.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function getTimestamp() { - return new Date().getTime(); -} diff --git a/src/utils/groupNotifies.ts b/src/utils/groupNotifies.ts index ef38eea..d468dbd 100644 --- a/src/utils/groupNotifies.ts +++ b/src/utils/groupNotifies.ts @@ -1,17 +1,17 @@ import _ from "lodash"; -import { UserLogType } from "@repositories/models/user-log"; +import { UserLog, UserLogType } from "@repositories/models/user-log"; -import { NotifyPair } from "@notifiers/type"; +import { UserLogMap } from "@notifiers/type"; -type MapKeys = Exclude; - -export function groupNotifies(pairs: NotifyPair[]): Record { - const result = _.groupBy(pairs, ([, log]) => log.type); +export function groupNotifies(pairs: UserLog[]): UserLogMap { + const result = _.groupBy(pairs, log => log.type); return { [UserLogType.Follow]: result[UserLogType.Follow] || [], [UserLogType.Unfollow]: result[UserLogType.Unfollow] || [], + [UserLogType.RenameUserId]: result[UserLogType.RenameUserId] || [], + [UserLogType.RenameDisplayName]: result[UserLogType.RenameDisplayName] || [], [UserLogType.Rename]: [ ...(result[UserLogType.RenameDisplayName] || []), ...(result[UserLogType.RenameUserId] || []), diff --git a/src/utils/index.ts b/src/utils/index.ts new file mode 100644 index 0000000..7a78f5d --- /dev/null +++ b/src/utils/index.ts @@ -0,0 +1,18 @@ +export * from "./buildQueryString"; +export * from "./cli"; +export * from "./config"; +export * from "./config.const"; +export * from "./config.type"; +export * from "./fetcher"; +export * from "./getDiff"; +export * from "./groupNotifies"; +export * from "./httpError"; +export * from "./isRequired"; +export * from "./logger"; +export * from "./mapBy"; +export * from "./measureTime"; +export * from "./noop"; +export * from "./parseCookie"; +export * from "./sleep"; +export * from "./throttle"; +export * from "./types"; diff --git a/src/utils/isRequired.ts b/src/utils/isRequired.ts new file mode 100644 index 0000000..ad16c1f --- /dev/null +++ b/src/utils/isRequired.ts @@ -0,0 +1,3 @@ +export function isRequired(value: T | undefined | null): value is T { + return Boolean(value); +} diff --git a/src/utils/logger.ts b/src/utils/logger.ts index d15a10d..f1c073a 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -16,7 +16,7 @@ const LOG_LEVEL_COLOR_MAP: Record> = { debug: chalk.magenta, }; -interface WorkOptions { +export interface WorkOptions { level: LogLevel; message: string; failedLevel?: LogLevel; diff --git a/src/utils/types.ts b/src/utils/types.ts index 4992fa6..8a6ac58 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -3,7 +3,6 @@ import { Logger } from "@utils/logger"; export type SelectOnly = { [Key in keyof Required as Required[Key] extends Type ? Key : never]: Record[Key]; }; - export type TypeMap = { [TKey in T["type"]]: TKey extends T["type"] ? Extract : never; }; diff --git a/src/watchers/base.ts b/src/watchers/base.ts index 2566d8b..850f13b 100644 --- a/src/watchers/base.ts +++ b/src/watchers/base.ts @@ -1,13 +1,17 @@ import pluralize from "pluralize"; -import { UserData } from "@repositories/models/user"; +import { User } from "@repositories/models/user"; import { Loggable } from "@utils/types"; +import { BaseEntity } from "typeorm"; export interface BaseWatcherOptions { type: TWatcher extends BaseWatcher ? Lowercase : string; } -export type PartialUserData = Omit; +export type PartialUser = Omit< + User, + keyof BaseEntity | "from" | "id" | "lastlyCheckedAt" | "createdAt" | "updatedAt" | "userLogs" | "userLogIds" +>; export abstract class BaseWatcher extends Loggable { protected constructor(name: TType) { @@ -16,17 +20,21 @@ export abstract class BaseWatcher extends Loggable public abstract initialize(): Promise; - public async doWatch(): Promise { + public async doWatch(startedAt: Date): Promise { const followers = await this.getFollowers(); + const from = this.getName().toLowerCase(); this.logger.info("Successfully crawled {} {}", [followers.length, pluralize("follower", followers.length)]); - return followers.map(user => ({ - ...user, - from: this.getName().toLowerCase(), - })); + return followers.map(user => + User.create({ + ...user, + id: `${from}:${user.uniqueId}`, + from, + lastlyCheckedAt: startedAt, + }), + ); } - protected abstract getFollowers(): Promise; - public abstract getProfileUrl(user: UserData): string; + protected abstract getFollowers(): Promise; } diff --git a/src/watchers/github/index.ts b/src/watchers/github/index.ts index dfa8154..1ef611d 100644 --- a/src/watchers/github/index.ts +++ b/src/watchers/github/index.ts @@ -1,16 +1,19 @@ import nodeFetch from "node-fetch"; import { Client, CombinedError, createClient } from "@urql/core"; -import { BaseWatcher, BaseWatcherOptions, PartialUserData } from "@watchers/base"; +import { BaseWatcher, BaseWatcherOptions, PartialUser } from "@watchers/base"; import { FollowersDocument, MeDocument } from "@watchers/github/queries"; import { FollowersQuery, FollowersQueryVariables, MeQuery } from "@root/queries.data"; +import { isRequired } from "@utils/isRequired"; import { Nullable } from "@utils/types"; export interface GitHubWatcherOptions extends BaseWatcherOptions { authToken: string; } +const isPartialUser = (user: PartialUser | null): user is PartialUser => Boolean(user); + export class GitHubWatcher extends BaseWatcher<"GitHub"> { private client: Client; @@ -35,13 +38,10 @@ export class GitHubWatcher extends BaseWatcher<"GitHub"> { public async initialize() { return; } - public getProfileUrl(user: PartialUserData) { - return `https://github.com/${user.userId}`; - } protected async getFollowers() { try { - const result: PartialUserData[] = []; + const result: PartialUser[] = []; const currentUserId = await this.getCurrentUserId(); let cursor: string | undefined = undefined; @@ -64,21 +64,17 @@ export class GitHubWatcher extends BaseWatcher<"GitHub"> { } else if (e.graphQLErrors && e.graphQLErrors.length > 0) { throw e.graphQLErrors[0]; } - } else { - throw e; } - } - return []; + throw e; + } } private async getCurrentUserId() { const { data, error } = await this.client.query(MeDocument, {}).toPromise(); if (error) { throw error; - } - - if (!data) { + } else if (!data) { throw new Error("No data returned from Me query"); } @@ -87,44 +83,28 @@ export class GitHubWatcher extends BaseWatcher<"GitHub"> { private async getFollowersFromUserId( targetUserId: string, cursor?: string, - ): Promise<[PartialUserData[], Nullable]> { + ): Promise<[PartialUser[], Nullable]> { const { data, error } = await this.client - .query(FollowersDocument, { - username: targetUserId, - cursor, - }) + .query(FollowersDocument, { username: targetUserId, cursor }) .toPromise(); if (error) { throw error; - } - - if (!data) { - throw new Error("No data returned from Followers query"); - } - - if (!data.user) { - throw new Error("No user returned from Followers query"); - } - - if (!data.user.followers.edges) { + } else if (!data?.user?.followers?.edges) { throw new Error("No followers returned from Followers query"); } return [ data.user.followers.edges - .map(edge => { - if (!edge?.node) { - return null; - } - - return { - uniqueId: edge.node.id, - userId: edge.node.login, - displayName: edge.node.name || edge.node.login, - }; - }) - .filter((item: PartialUserData | null): item is PartialUserData => Boolean(item)), + .map(edge => edge?.node) + .filter(isRequired) + .map(node => ({ + uniqueId: node.id, + userId: node.login, + displayName: node.name || node.login, + profileUrl: `https://github.com/${node.login}`, + })) + .filter(isPartialUser), data.user.followers.pageInfo.endCursor, ]; } diff --git a/src/watchers/twitter/index.ts b/src/watchers/twitter/index.ts index cf2b839..a7055a8 100644 --- a/src/watchers/twitter/index.ts +++ b/src/watchers/twitter/index.ts @@ -1,9 +1,7 @@ import { TwitterApi, TwitterApiReadOnly } from "twitter-api-v2"; import { TwitterApiRateLimitPlugin } from "@twitter-api-v2/plugin-rate-limit"; -import { UserData } from "@repositories/models/user"; - -import { BaseWatcher, BaseWatcherOptions } from "@watchers/base"; +import { BaseWatcher, BaseWatcherOptions, PartialUser } from "@watchers/base"; export interface TwitterWatcherOptions extends BaseWatcherOptions { bearerToken: string; @@ -33,11 +31,8 @@ export class TwitterWatcher extends BaseWatcher<"Twitter"> { this.logger.verbose("Successfully initialized with user id {}", [data.id]); this.currentUserId = data.id; } - public getProfileUrl(user: UserData) { - return `https://twitter.com/${user.userId}`; - } - protected async getFollowers() { + protected async getFollowers(): Promise { if (!this.currentUserId) { throw new Error("Watcher is not initialized"); } @@ -51,6 +46,7 @@ export class TwitterWatcher extends BaseWatcher<"Twitter"> { uniqueId: user.id, displayName: user.name, userId: user.username, + profileUrl: `https://twitter.com/${user.username}`, })); } } diff --git a/tsconfig.json b/tsconfig.json index 645de76..2eed051 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -27,12 +27,15 @@ "paths": { "@root/*": ["./src/*"], "@utils/*": ["./src/utils/*"], + "@utils": ["./src/utils"], "@watchers/*": ["./src/watchers/*"], "@watchers": ["./src/watchers"], "@followers/*": ["./src/followers/*"], "@repositories/*": ["./src/repositories/*"], "@notifiers/*": ["./src/notifiers/*"], - "@notifiers": ["./src/notifiers"] + "@notifiers": ["./src/notifiers"], + "@tasks/*": ["./src/tasks/*"], + "@tasks": ["./src/tasks"] } } }