feat: update project configuration and implement base service class f…

…or logging and configuration management

.gitignore

@@ -1,4 +1,5 @@
 node_modules/
+node_modules
 logs/
 dist/
 config.toml

.tool-versions

@@ -1,3 +1,3 @@
 nodejs 18.16.0
-pnpm 6.24.4
+pnpm 9.15.2
 python 3.12.0

core/base.ts

@@ -0,0 +1,8 @@
+import type { Logger } from 'winston';
+import { config } from '@core/config';
+import { logger } from '@core/logger';
+
+export class ServiceBase {
+  public log: Logger = logger;
+  public config = config;
+}

core/bot.ts

@@ -6,36 +6,22 @@
 import { Mwn, type MwnOptions } from "mwn";
 import { version, dependencies } from '../package.json'
 import { Command } from "@commander-js/extra-typings"
+import { Replica } from '@core/replica';
 import { Cron, type CronOptions } from 'croner';
 import chalk from 'chalk';
-import { Replica } from './replica';
+import { ServiceBase } from './base';
 
-interface ScheduleOptions {
-  pattern: string | Date;
-  options: CronOptions;
-}
-
-if (!Bun.file('../config.toml').exists()) {
-  throw new Error('Please create config.toml')
-}
-
-const config = await import('../config.toml')
-
-if (!config.oauth.consumerToken || !config.oauth.consumerSecret || !config.oauth.accessToken || !config.oauth.accessSecret) {
-  throw new Error('Please fill in the OAuth credentials in config.toml')
-}
-
-export class Bot {
+export class Bot extends ServiceBase {
   private _botOptions: MwnOptions = {
-    apiUrl: config.bot.apiUrl || "https://th.wikipedia.org/w/api.php",
+    apiUrl: this.config.bot.apiUrl || "https://th.wikipedia.org/w/api.php",
     OAuthCredentials: {
-      consumerToken: config.oauth.consumerToken || process.env.BOT_CONSUMER_TOKEN,
-      consumerSecret: config.oauth.consumerSecret || process.env.BOT_CONSUMER_SECRET,
-      accessToken: config.oauth.accessToken || process.env.BOT_ACCESS_TOKEN,
-      accessSecret: config.oauth.accessSecret || process.env.BOT_ACCESS_SECRET,
+      consumerToken: this.config.oauth.consumerToken,
+      consumerSecret: this.config.oauth.consumerSecret,
+      accessToken: this.config.oauth.accessToken,
+      accessSecret: this.config.oauth.accessSecret,
     },
     // Set your user agent (required for WMF wikis, see https://meta.wikimedia.org/wiki/User-Agent_policy):
-    userAgent: `${config.bot.username}/${version} (${config.bot.contact}) mwn/${dependencies.mwn}`,
+    userAgent: `${this.config.bot.username}/${version} (${this.config.bot.contact}) mwn/${dependencies.mwn}`,
     defaultParams: {
       assert: "user",
     },
@@ -49,10 +35,6 @@ export class Bot {
 
   public job?: Cron
 
-  public log(obj: any) {
-    return Mwn.log(obj)
-  }
-
   public bot = new Mwn({
     ...this._botOptions
   });
@@ -62,6 +44,7 @@ export class Bot {
   public replica = new Replica()
 
   constructor() {
+    super()
     this.info = {
       id: "bot",
       name: "Bot",
@@ -102,10 +85,13 @@ export class Bot {
     }
   }
 
-  async schedule(options: ScheduleOptions) {
+  async schedule(options: {
+    pattern: string | Date;
+    options: CronOptions;
+  }) {
     this.job = new Cron(options.pattern, {
       name: this.info.id,
-      timezone: config.bot.timezone,
+      timezone: this.config.bot.timezone,
       ...options.options
     })
     this.job.schedule(() => this.run())

core/config.ts

@@ -0,0 +1,46 @@
+import { z } from 'zod'
+
+if (!Bun.file('../config.toml').exists()) {
+  throw new Error('Please create config.toml')
+}
+
+export const config = await z.object({
+  oauth: z.object({
+    consumerToken: z.string(),
+    consumerSecret: z.string(),
+    accessToken: z.string(),
+    accessSecret: z.string(),
+  }),
+  bot: z.object({
+    apiUrl: z.string(),
+    username: z.string(),
+    contact: z.string(),
+    timezone: z.string(),
+  }),
+  toolforge: z.object({
+    login: z.string(),
+    tooluser: z.string(),
+    deploykey: z.string(),
+  }),
+  replica: z.object({
+    username: z.string(),
+    password: z.string(),
+    dbname: z.string(),
+    cluster: z.string(),
+    port: z.number(),
+  }),
+  scripts: z.object({
+    archive: z.object({
+      key_salt: z.string().optional(),
+    }),
+  }),
+  logger: z.object({
+    logPath: z.string(),
+    level: z.string().default("info"),
+  }),
+  discord: z.object({
+    logger: z.object({
+      webhook: z.string().optional(),
+    }),
+  }),
+}).parseAsync(await import('../config.toml'))

core/logger.ts

@@ -0,0 +1,36 @@
+import { createLogger, transports, format } from 'winston';
+import { config } from '@core/config';
+import chalk from 'chalk';
+
+export const logger = createLogger({
+  level: config.logger.level,
+  format: format.combine(
+    format.timestamp(),
+    format.errors({
+      stack: true,
+    }),
+    format.splat(),
+    format.json(),
+  ),
+  transports: [
+    new transports.File({
+      filename: `${config.logger.logPath}/output.log`,
+      maxsize: 5242880, // 5MB
+    }),
+    new transports.File({ filename: 'error.log', level: 'error' }),
+    new transports.Console({
+      format: format.combine(
+        format.colorize(),
+        format.printf(({ level, message, timestamp, durationMs }) => {
+          return `${chalk.dim(`[${timestamp}]`)} ${level}: ${message}${durationMs ? ` ${chalk.dim(`(${durationMs}ms)`)}` : ''}`;
+        })
+      )
+    }),
+  ],
+  exceptionHandlers: [
+    new transports.File({ filename: `${config.logger.logPath}/exceptions.log` })
+  ],
+  rejectionHandlers: [
+    new transports.File({ filename: 'rejections.log' })
+  ]
+});

core/replica.ts

@@ -1,15 +1,7 @@
 import mysql from "mysql2/promise";
-import { $, type Server } from "bun"
-
-const config = await import('../config.toml')
-
-if (!config.replica) {
-  throw new Error("No replica config found.");
-}
-
-if (!config.replica.username || !config.replica.password) {
-  throw new Error("Please fill in the replica credentials in config.toml");
-}
+import { $ } from "bun"
+import { ServiceBase } from '@core/base';
+import { config } from '@core/config';
 
 function getReplicaHost(dbname: string, cluster: string = "web") {
   if (!['web', 'analytics'].includes(cluster)) {
@@ -27,22 +19,24 @@ function getReplicaHost(dbname: string, cluster: string = "web") {
 }
 
 
-export class Replica {
+export class Replica extends ServiceBase {
   private _replicaOptions: mysql.PoolOptions = {
-    user: config.replica.username,
-    password: config.replica.password,
-    port: Number(config.replica.port || 3306),
+    user: this.config.replica.username,
+    password: this.config.replica.password,
+    port: Number(this.config.replica.port || 3306),
     waitForConnections: true,
   }
 
   public conn: mysql.Connection | null = null
 
-  constructor() { }
+  constructor() {
+    super()
+  }
 
   public async init() {
     if (!this.isRunOnToolforge()) {
       console.warn("Not running on Toolforge, don't forget to set up SSH tunnel using `. replica-tunnel` in separate terminal.")
-      const database = config.replica.dbname
+      const database = this.config.replica.dbname
       this._replicaOptions = {
         ...this._replicaOptions,
         host: '127.0.0.1',
@@ -51,16 +45,16 @@ export class Replica {
     } else {
       this._replicaOptions = {
         ...this._replicaOptions,
-        host: getReplicaHost(config.replica.dbname),
-        database: config.replica.dbname,
+        host: getReplicaHost(this.config.replica.dbname),
+        database: this.config.replica.dbname,
       }
     }
     this.conn = await mysql.createConnection(this._replicaOptions)
   }
 
   public static async createReplicaTunnel(dbname: string, cluster: string = "web", port: number = 3306) {
     if (!config.toolforge.login) {
-      throw new Error('Please fill in the Toolforge login in config.toml')
+      throw new Error('Please fill in the Toolforge login in this.config.toml')
     }
 
     const host = getReplicaHost(dbname, cluster)
@@ -71,7 +65,7 @@ export class Replica {
   }
 
   private isRunOnToolforge() {
-    return process.env.USER == config.toolforge.tooluser
+    return process.env.USER == this.config.toolforge.tooluser
   }
 
   public async query(sql: string, values: any[] = []) {

core/run.ts

@@ -1,24 +1,34 @@
+import { Bot } from '@core/bot'
+import { Replica } from '@core/replica'
+import { createId } from '@paralleldrive/cuid2'
 import { Command, InvalidArgumentError, Option } from '@commander-js/extra-typings'
-import { Bot } from './bot'
 import { version } from "../package.json"
-import { $ } from "bun"
-import { Replica } from './replica'
+import { ServiceBase } from './base'
 
-class ScriptRunner {
+class ScriptRunner extends ServiceBase {
   private cli = new Command()
 
   async scriptModule(scriptName: string) {
     if (!scriptName) {
       throw new Error('No script name provided')
     }
-    if (!scriptName.match(/^[a-z0-9-]+$/)) {
+    if (!scriptName.match(/^[a-z0-9-\/]+$/)) {
       throw new Error('Invalid script name')
     }
-    if (!Bun.file(`./scripts/${scriptName}.ts`).exists()) {
+    if (!Bun.file(`@scripts/${scriptName}.ts`).exists()) {
       throw new Error('Script not found')
     }
 
-    const scriptModule = await import(`../scripts/${scriptName}.ts`)
+    const scriptModule = await import(`@scripts/${scriptName}.ts`)
+
+    if (!scriptModule.default) {
+      throw new Error('Script must have a default export')
+    }
+
+    // check if scriptModule is a Bot instance
+    if (!(Object.getPrototypeOf(scriptModule.default) === Bot || Object.getPrototypeOf(scriptModule.default.prototype) === Bot)) {
+      throw new Error('Script must be a Bot instance')
+    }
 
     return (new scriptModule.default) as unknown as Bot
   }
@@ -39,7 +49,17 @@ class ScriptRunner {
         scriptModule.cli
           .name('run ' + scriptName)
           .description(scriptModule.scriptDescription)
+        // Global cli options
+        scriptModule.cli.option("-v, --verbose", "Show debug logging")
         scriptModule.cli.parse(process.argv.slice(2))
+        scriptModule.log.defaultMeta = {
+          script: scriptName,
+          rid: createId(),
+        }
+        // @ts-ignore
+        if (scriptModule.cli.opts().verbose) {
+          scriptModule.log.level = 'debug'
+        }
         try {
           await scriptModule.beforeRun()
           await scriptModule.run()

package.json

@@ -15,24 +15,25 @@
   "dependencies": {
     "@commander-js/extra-typings": "^13.0.0",
     "@paralleldrive/cuid2": "^2.2.2",
-    "bun": "^1.1.42",
     "chalk": "^5.4.1",
+    "commander": "^13.0.0",
     "cors": "^2.8.5",
     "croner": "^9.0.0",
     "cronstrue": "^2.52.0",
     "moment": "^2.30.1",
     "mwn": "^2.0.4",
     "mysql2": "^3.12.0",
-    "winston": "^3.17.0"
+    "winston": "^3.17.0",
+    "zod": "^3.24.1"
   },
   "devDependencies": {
     "@types/bun": "^1.1.14",
     "@types/cors": "^2.8.17",
-    "@types/cron": "^2.4.3",
     "@types/express": "^5.0.0",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.10.3",
     "@types/ssh2": "^1.15.1",
+    "bun": "^1.1.42",
     "husky": "^9.1.7",
     "lint-staged": "^15.3.0",
     "prettier": "3.4.2",
@@ -59,9 +60,10 @@
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "format": "prettier --write . --ignore-unknown",
-    "start": "bun core/web.ts",
-    "build": "sh ./build.sh",
+    "build": "bun i",
+    "start": "bun ./core/web.ts",
     "prepare": "husky"
   },
-  "type": "module"
+  "type": "module",
+  "packageManager": "[email protected]"
 }