From 265af322ff1b57987e26d7d56fb38439ae62cedf Mon Sep 17 00:00:00 2001 From: sorja Date: Mon, 30 Sep 2024 10:23:15 +0300 Subject: [PATCH 1/9] 3931 - yarn migration-steps:reset --- package.json | 1 + src/test/migrations/reset.ts | 65 ++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 src/test/migrations/reset.ts diff --git a/package.json b/package.json index 13bfa3fb9a..08f8e76f70 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "migration-public:create": "src/tools/migrations/public/create-migration-step.sh", "migration-public:run": "ts-node src/tools/migrations/public", "migration-steps:create": "src/test/migrations/create-migration-step.sh", + "migration-steps:reset": "ts-node src/test/migrations/reset.ts", "migration-steps:run": "ts-node src/test/migrations/", "migration-steps:watch": "ts-node-dev --respawn --transpile-only --exit-child --watch src/test/migrations/ src/test/migrations/ -- --watch", "run:dev": "run-p run:dev:client run:dev:server", diff --git a/src/test/migrations/reset.ts b/src/test/migrations/reset.ts new file mode 100644 index 0000000000..2d6695717e --- /dev/null +++ b/src/test/migrations/reset.ts @@ -0,0 +1,65 @@ +import 'tsconfig-paths/register' +import 'dotenv/config' + +import * as fs from 'fs' +import * as path from 'path' +import * as readline from 'readline' + +import { DB } from 'server/db' +import { Logger } from 'server/utils/logger' + +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, +}) + +const _confirmReset = (): Promise => { + return new Promise((resolve) => { + rl.question('Type "reset" to proceed with migration-steps reset: ', (answer) => { + resolve(answer.toLowerCase() === 'reset') + }) + }) +} + +const resetMigrationSteps = async () => { + try { + const confirmed = await _confirmReset() + if (!confirmed) { + Logger.info('Reset operation cancelled') + return + } + + const stepsDir = path.join(__dirname, 'steps') + const files = fs.readdirSync(stepsDir) + files.forEach((file) => { + if (file !== 'template.ts' && file.endsWith('.ts')) { + fs.unlinkSync(path.join(stepsDir, file)) + Logger.info(`Removed migration file: ${file}`) + } + }) + + // Truncate migration_steps table and reset the primary key + await DB.tx(async (t) => { + await t.none('TRUNCATE TABLE migrations.steps RESTART IDENTITY') + Logger.info('Truncated migrations.steps table and reset primary key') + }) + + Logger.info('Migration steps reset completed successfully') + } catch (error) { + Logger.error('Error resetting migration steps:', error) + throw error + } finally { + rl.close() + await DB.$pool.end() + } +} + +Logger.info('Starting migration steps reset') +resetMigrationSteps() + .then(() => { + Logger.info('Migration steps reset finished') + process.exit(0) + }) + .catch(() => { + process.exit(1) + }) From 47d82510b20f4ad5c60d88e9f4a46801e12d1365 Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 09:32:09 +0300 Subject: [PATCH 2/9] 3931 - createMigrationStep: use TypeScript --- package.json | 2 +- src/test/migrations/create-migration-step.sh | 19 ----- src/test/migrations/createMigrationStep.ts | 76 ++++++++++++++++++++ src/test/migrations/utils/index.ts | 30 ++++++++ 4 files changed, 107 insertions(+), 20 deletions(-) delete mode 100755 src/test/migrations/create-migration-step.sh create mode 100644 src/test/migrations/createMigrationStep.ts create mode 100644 src/test/migrations/utils/index.ts diff --git a/package.json b/package.json index 08f8e76f70..e18e7499cb 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "heroku-cleanup": "cd node_modules/bcrypt && node-pre-gyp install --fallback-to-build", "migration-public:create": "src/tools/migrations/public/create-migration-step.sh", "migration-public:run": "ts-node src/tools/migrations/public", - "migration-steps:create": "src/test/migrations/create-migration-step.sh", + "migration-steps:create": "ts-node src/test/migrations/createMigrationStep.ts", "migration-steps:reset": "ts-node src/test/migrations/reset.ts", "migration-steps:run": "ts-node src/test/migrations/", "migration-steps:watch": "ts-node-dev --respawn --transpile-only --exit-child --watch src/test/migrations/ src/test/migrations/ -- --watch", diff --git a/src/test/migrations/create-migration-step.sh b/src/test/migrations/create-migration-step.sh deleted file mode 100755 index 4900f4a83f..0000000000 --- a/src/test/migrations/create-migration-step.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ] - then - echo "Give the name of the migration step as parameter" - exit 2 -fi - -. .env -_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -_DATE=$(date +%Y%m%d%H%M%S) -_FILENAME=$_DATE-step-$1.ts -_FILEPATH=$_DIR/steps/$_FILENAME - -cp "$_DIR/steps/template.ts" "$_FILEPATH" - -sed -i '' "s/#NAME#/$1/g" "$_FILEPATH" - -echo "Created migration step $_FILEPATH" diff --git a/src/test/migrations/createMigrationStep.ts b/src/test/migrations/createMigrationStep.ts new file mode 100644 index 0000000000..6addd45eea --- /dev/null +++ b/src/test/migrations/createMigrationStep.ts @@ -0,0 +1,76 @@ +import 'tsconfig-paths/register' +import 'dotenv/config' + +import * as fs from 'fs' +import * as path from 'path' +import * as readline from 'readline' + +import { Logger } from 'server/utils/logger' + +/** + * Ask the user for the name of the migration step + * @returns Migration step name + */ +const askStepName = (): Promise => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }) + + return new Promise((resolve) => { + rl.question('Please enter a name for the migration step: ', (answer) => { + rl.close() + resolve(answer.trim()) + }) + }) +} + +/** + * Get the current date in the format YYYY-MM-DD-HH-MM-SS + * @returns The current date in the format YYYY-MM-DD-HH-MM-SS + */ +const getDate = (): string => { + const date = new Date() + const year = date.getFullYear() + const month = (date.getMonth() + 1).toString().padStart(2, '0') + const day = date.getDate().toString().padStart(2, '0') + const hour = date.getHours().toString().padStart(2, '0') + const minute = date.getMinutes().toString().padStart(2, '0') + const second = date.getSeconds().toString().padStart(2, '0') + return `${year}${month}${day}${hour}${minute}${second}` +} + +export const createMigrationStep = async (initialStepName: string): Promise<{ fileName: string; filePath: string }> => { + let stepName = initialStepName + if (!stepName) { + stepName = await askStepName() + if (!stepName) { + Logger.error('Migration step name cannot be empty. Exiting.') + process.exit(1) + } + } + + const currentDir = __dirname + + const fileName = `${getDate()}-step-${stepName}.ts` + const filePath = path.join(currentDir, 'steps', fileName) + + const template = stepName === 'reset' ? 'template-reset.ts' : 'template.ts' + const templatePath = path.join(currentDir, 'steps', template) + + try { + let content = fs.readFileSync(templatePath, 'utf8') + content = content.replace(/#NAME#/g, stepName) + + fs.writeFileSync(filePath, content) + Logger.info(`Created migration step ${filePath}`) + } catch (error) { + Logger.error('Error creating migration step:', error) + process.exit(1) + } + + return { fileName, filePath } +} + +const stepName = process.argv[2] +createMigrationStep(stepName) diff --git a/src/test/migrations/utils/index.ts b/src/test/migrations/utils/index.ts new file mode 100644 index 0000000000..a44e9e8781 --- /dev/null +++ b/src/test/migrations/utils/index.ts @@ -0,0 +1,30 @@ +import * as fs from 'fs' +import * as path from 'path' + +/** + * Get the migration files from the migration steps directory + * @param includeResetSteps - Whether to include reset steps in the list + * @returns The migration files + */ +export const getMigrationFiles = (includeResetSteps = false): string[] => { + const stepsDir = path.join(__dirname, 'steps') + return fs.readdirSync(stepsDir).filter((file) => { + const isResetStep = file.includes('step-reset') + return ( + file.endsWith('.ts') && + file !== 'template.ts' && + file !== 'template-reset.ts' && + (includeResetSteps || !isResetStep) + ) + }) +} + +/** + * Get the files to remove from the migration steps directory + * @param latestResetStep - The name of the latest reset step + * @returns The files to remove, excluding the latest reset step + */ +export const getFilesToRemove = (latestResetStep: string): string[] => { + const allFiles = getMigrationFiles(true) + return allFiles.filter((file) => file !== latestResetStep) +} From dd2c1f0191d834d9b257ab739edd5c7afbe23c5a Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 09:32:29 +0300 Subject: [PATCH 3/9] 3931 - createMigrationStep: use TypeScript --- src/test/migrations/steps/template-reset.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/test/migrations/steps/template-reset.ts diff --git a/src/test/migrations/steps/template-reset.ts b/src/test/migrations/steps/template-reset.ts new file mode 100644 index 0000000000..54a5a0f110 --- /dev/null +++ b/src/test/migrations/steps/template-reset.ts @@ -0,0 +1,19 @@ +import { BaseProtocol } from 'server/db' +import { Logger } from 'server/utils/logger' + +export default async (client: BaseProtocol, fileName: string) => { + try { + // === 1. Truncate migration_steps table and reset the primary key + await client.none('truncate table migrations.steps restart identity') + Logger.info('Truncated migrations.steps table and reset primary key') + + // === 2. Insert the current reset step into the migrations.steps table + await client.none('insert into migrations.steps (name) values ($1)', [fileName]) + Logger.info(`Inserted reset step ${fileName} into migrations.steps table`) + + Logger.info('Database table migrations.steps reset completed successfully') + } catch (error) { + Logger.error('Error resetting database:', error) + throw error + } +} From c4ed55db9b2b0b1374e1215d6b30ce52f282d495 Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 09:34:45 +0300 Subject: [PATCH 4/9] 3931 - migration-steps:reset: check all migrations have ran --- src/test/migrations/index.ts | 9 ++---- src/test/migrations/reset.ts | 57 +++++++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/test/migrations/index.ts b/src/test/migrations/index.ts index 3a55383cbc..44af0924f8 100644 --- a/src/test/migrations/index.ts +++ b/src/test/migrations/index.ts @@ -1,9 +1,6 @@ import 'tsconfig-paths/register' import 'dotenv/config' -import * as fs from 'fs' -import * as path from 'path' - import { VisitCycleLinksQueueFactory } from 'server/controller/cycleData/links/visitCycleLinks/queueFactory' import { WorkerFactory as VisitLinksWorkerFactory } from 'server/controller/cycleData/links/visitCycleLinks/workerFactory' import { UpdateDependenciesQueueFactory } from 'server/controller/cycleData/updateDependencies/queueFactory' @@ -12,6 +9,8 @@ import { DB } from 'server/db' import { RedisData } from 'server/repository/redis/redisData' import { Logger } from 'server/utils/logger' +import { getMigrationFiles } from './utils' + const client = DB let migrationSteps: Array let previousMigrations: Array = [] @@ -38,9 +37,7 @@ const tableDDL = ` const init = async () => { await client.query(tableDDL) previousMigrations = await client.map('select * from migrations.steps', [], (row) => row.name) - migrationSteps = fs - .readdirSync(path.join(__dirname, `steps`)) - .filter((file) => file !== 'template.ts' && file.endsWith('.ts') && !previousMigrations.includes(file)) + migrationSteps = getMigrationFiles(true).filter((file) => !previousMigrations.includes(file)) } const close = async () => { diff --git a/src/test/migrations/reset.ts b/src/test/migrations/reset.ts index 2d6695717e..39945c7f20 100644 --- a/src/test/migrations/reset.ts +++ b/src/test/migrations/reset.ts @@ -8,6 +8,9 @@ import * as readline from 'readline' import { DB } from 'server/db' import { Logger } from 'server/utils/logger' +import { createMigrationStep } from './createMigrationStep' +import { getFilesToRemove, getMigrationFiles } from './utils' + const rl = readline.createInterface({ input: process.stdin, output: process.stdout, @@ -21,27 +24,59 @@ const _confirmReset = (): Promise => { }) } +const checkAllMigrationsRan = async () => { + const files = getMigrationFiles() + + const dbMigrations = await DB.manyOrNone('select * from migrations.steps;') + + const missingMigrations = files.filter((file) => !dbMigrations.some((dbMigration) => dbMigration.name === file)) + + if (missingMigrations.length > 0) { + Logger.warn('The following migrations have not been run:') + missingMigrations.forEach((file) => Logger.warn(`- ${file}`)) + return false + } + + return true +} + +const deleteOldMigrationFiles = (latestResetStep: string) => { + const stepsDir = path.join(__dirname, 'steps') + const filesToRemove = getFilesToRemove(latestResetStep) + + filesToRemove.forEach((file) => { + fs.unlinkSync(path.join(stepsDir, file)) + Logger.info(`Removed migration file: ${file}`) + }) +} + const resetMigrationSteps = async () => { try { + // === 1. Check if all migrations ran + const allMigrationsRan = await checkAllMigrationsRan() + if (!allMigrationsRan) { + Logger.error('Not all migrations have been run. Please run all migrations before resetting.') + return + } + + // === 2. Confirm reset from user const confirmed = await _confirmReset() if (!confirmed) { Logger.info('Reset operation cancelled') return } - const stepsDir = path.join(__dirname, 'steps') - const files = fs.readdirSync(stepsDir) - files.forEach((file) => { - if (file !== 'template.ts' && file.endsWith('.ts')) { - fs.unlinkSync(path.join(stepsDir, file)) - Logger.info(`Removed migration file: ${file}`) - } - }) + // === 3. Create reset step + const { filePath, fileName } = await createMigrationStep('reset') + // eslint-disable-next-line @typescript-eslint/no-var-requires,global-require,import/no-dynamic-require + const resetStep = require(filePath).default - // Truncate migration_steps table and reset the primary key await DB.tx(async (t) => { - await t.none('TRUNCATE TABLE migrations.steps RESTART IDENTITY') - Logger.info('Truncated migrations.steps table and reset primary key') + // === 4. Run reset step + await resetStep(t, fileName) + + // === 5. Delete old migration files + deleteOldMigrationFiles(fileName) }) Logger.info('Migration steps reset completed successfully') From d3f674a01d41ba1be5e18688c7d2772b617b0bf9 Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 09:36:35 +0300 Subject: [PATCH 5/9] fix typo in comment --- src/test/migrations/createMigrationStep.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/migrations/createMigrationStep.ts b/src/test/migrations/createMigrationStep.ts index 6addd45eea..c6c59c809d 100644 --- a/src/test/migrations/createMigrationStep.ts +++ b/src/test/migrations/createMigrationStep.ts @@ -26,8 +26,8 @@ const askStepName = (): Promise => { } /** - * Get the current date in the format YYYY-MM-DD-HH-MM-SS - * @returns The current date in the format YYYY-MM-DD-HH-MM-SS + * Get the current date in the format YYYYMMDDHHMMSS + * @returns The current date in the format YYYYMMDDHHMMSS */ const getDate = (): string => { const date = new Date() From 1f40b4f4bb11ec4f6d007c6e2ba22322fb86cacd Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 09:43:39 +0300 Subject: [PATCH 6/9] 3931 - migration-step utils: fix steps path --- src/test/migrations/utils/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/migrations/utils/index.ts b/src/test/migrations/utils/index.ts index a44e9e8781..1db0903189 100644 --- a/src/test/migrations/utils/index.ts +++ b/src/test/migrations/utils/index.ts @@ -6,8 +6,8 @@ import * as path from 'path' * @param includeResetSteps - Whether to include reset steps in the list * @returns The migration files */ -export const getMigrationFiles = (includeResetSteps = false): string[] => { - const stepsDir = path.join(__dirname, 'steps') +export const getMigrationFiles = (includeResetSteps = false): Array => { + const stepsDir = path.join(__dirname, '..', 'steps') return fs.readdirSync(stepsDir).filter((file) => { const isResetStep = file.includes('step-reset') return ( @@ -24,7 +24,7 @@ export const getMigrationFiles = (includeResetSteps = false): string[] => { * @param latestResetStep - The name of the latest reset step * @returns The files to remove, excluding the latest reset step */ -export const getFilesToRemove = (latestResetStep: string): string[] => { +export const getFilesToRemove = (latestResetStep: string): Array => { const allFiles = getMigrationFiles(true) return allFiles.filter((file) => file !== latestResetStep) } From 95fd18c857b374ab9d53ad7b07a017be610d2df5 Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 10:11:48 +0300 Subject: [PATCH 7/9] migration-steps create: simplify getDate --- src/test/migrations/createMigrationStep.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/test/migrations/createMigrationStep.ts b/src/test/migrations/createMigrationStep.ts index c6c59c809d..8e54df80a9 100644 --- a/src/test/migrations/createMigrationStep.ts +++ b/src/test/migrations/createMigrationStep.ts @@ -30,14 +30,11 @@ const askStepName = (): Promise => { * @returns The current date in the format YYYYMMDDHHMMSS */ const getDate = (): string => { - const date = new Date() - const year = date.getFullYear() - const month = (date.getMonth() + 1).toString().padStart(2, '0') - const day = date.getDate().toString().padStart(2, '0') - const hour = date.getHours().toString().padStart(2, '0') - const minute = date.getMinutes().toString().padStart(2, '0') - const second = date.getSeconds().toString().padStart(2, '0') - return `${year}${month}${day}${hour}${minute}${second}` + const now = new Date() + return now + .toISOString() + .replace(/[-:T.]/g, '') + .slice(0, 14) } export const createMigrationStep = async (initialStepName: string): Promise<{ fileName: string; filePath: string }> => { From 9675b3df2b29c3b5c821cf3020d28d288ea4a8af Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 10:19:32 +0300 Subject: [PATCH 8/9] 3931 - fix reading user input --- src/test/migrations/createMigrationStep.ts | 15 +++------------ src/test/migrations/reset.ts | 18 ++++-------------- src/test/migrations/utils/index.ts | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/test/migrations/createMigrationStep.ts b/src/test/migrations/createMigrationStep.ts index 8e54df80a9..c07a32920b 100644 --- a/src/test/migrations/createMigrationStep.ts +++ b/src/test/migrations/createMigrationStep.ts @@ -3,26 +3,17 @@ import 'dotenv/config' import * as fs from 'fs' import * as path from 'path' -import * as readline from 'readline' import { Logger } from 'server/utils/logger' +import { getUserInput } from './utils' + /** * Ask the user for the name of the migration step * @returns Migration step name */ const askStepName = (): Promise => { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }) - - return new Promise((resolve) => { - rl.question('Please enter a name for the migration step: ', (answer) => { - rl.close() - resolve(answer.trim()) - }) - }) + return getUserInput('Please enter a name for the migration step: ') } /** diff --git a/src/test/migrations/reset.ts b/src/test/migrations/reset.ts index 39945c7f20..69698019fa 100644 --- a/src/test/migrations/reset.ts +++ b/src/test/migrations/reset.ts @@ -3,25 +3,16 @@ import 'dotenv/config' import * as fs from 'fs' import * as path from 'path' -import * as readline from 'readline' import { DB } from 'server/db' import { Logger } from 'server/utils/logger' import { createMigrationStep } from './createMigrationStep' -import { getFilesToRemove, getMigrationFiles } from './utils' +import { getFilesToRemove, getMigrationFiles, getUserInput } from './utils' -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, -}) - -const _confirmReset = (): Promise => { - return new Promise((resolve) => { - rl.question('Type "reset" to proceed with migration-steps reset: ', (answer) => { - resolve(answer.toLowerCase() === 'reset') - }) - }) +const _confirmReset = async (): Promise => { + const answer = await getUserInput('Type "reset" to proceed with migration-steps reset: ') + return answer.toLowerCase() === 'reset' } const checkAllMigrationsRan = async () => { @@ -84,7 +75,6 @@ const resetMigrationSteps = async () => { Logger.error('Error resetting migration steps:', error) throw error } finally { - rl.close() await DB.$pool.end() } } diff --git a/src/test/migrations/utils/index.ts b/src/test/migrations/utils/index.ts index 1db0903189..a640ded967 100644 --- a/src/test/migrations/utils/index.ts +++ b/src/test/migrations/utils/index.ts @@ -1,5 +1,25 @@ import * as fs from 'fs' import * as path from 'path' +import * as readline from 'readline' + +/** + * Get user input from the terminal + * @param question - The question to ask the user + * @returns The user input + */ +export const getUserInput = (question: string): Promise => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }) + + return new Promise((resolve) => { + rl.question(question, (answer) => { + rl.close() + resolve(answer.trim()) + }) + }) +} /** * Get the migration files from the migration steps directory From 3ca55ff40ddc3fc9711505ebb2e3707a55046eb1 Mon Sep 17 00:00:00 2001 From: sorja Date: Wed, 2 Oct 2024 10:22:02 +0300 Subject: [PATCH 9/9] 3931 - execute createMigrationStep only when called from terminal --- src/test/migrations/createMigrationStep.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/migrations/createMigrationStep.ts b/src/test/migrations/createMigrationStep.ts index c07a32920b..845b167de3 100644 --- a/src/test/migrations/createMigrationStep.ts +++ b/src/test/migrations/createMigrationStep.ts @@ -60,5 +60,7 @@ export const createMigrationStep = async (initialStepName: string): Promise<{ fi return { fileName, filePath } } -const stepName = process.argv[2] -createMigrationStep(stepName) +if (require.main === module) { + const stepName = process.argv[2] + createMigrationStep(stepName) +}