From bb8c59d7b7b1ad7ce2fccf3c4d56691a3e6b606b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mir=C3=B3=20Sorja?= Date: Fri, 4 Oct 2024 19:21:04 +0200 Subject: [PATCH] 3931 - yarn migration-steps:reset (#3979) * 3931 - yarn migration-steps:reset * 3931 - createMigrationStep: use TypeScript * 3931 - createMigrationStep: use TypeScript * 3931 - migration-steps:reset: check all migrations have ran * fix typo in comment * 3931 - migration-step utils: fix steps path * migration-steps create: simplify getDate * 3931 - fix reading user input * 3931 - execute createMigrationStep only when called from terminal --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- package.json | 3 +- src/test/migrations/create-migration-step.sh | 19 ----- src/test/migrations/createMigrationStep.ts | 66 ++++++++++++++ src/test/migrations/index.ts | 9 +- src/test/migrations/reset.ts | 90 ++++++++++++++++++++ src/test/migrations/steps/template-reset.ts | 19 +++++ src/test/migrations/utils/index.ts | 50 +++++++++++ 7 files changed, 230 insertions(+), 26 deletions(-) delete mode 100755 src/test/migrations/create-migration-step.sh create mode 100644 src/test/migrations/createMigrationStep.ts create mode 100644 src/test/migrations/reset.ts create mode 100644 src/test/migrations/steps/template-reset.ts create mode 100644 src/test/migrations/utils/index.ts diff --git a/package.json b/package.json index f3b0497c08..2a615d411b 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "heroku-cleanup": "cd node_modules/bcrypt && node-pre-gyp install --fallback-to-build", "migration-public:create": "src/tools/migrations/public/create-migration-step.sh", "migration-public:run": "ts-node src/tools/migrations/public", - "migration-steps:create": "src/test/migrations/create-migration-step.sh", + "migration-steps:create": "ts-node src/test/migrations/createMigrationStep.ts", + "migration-steps:reset": "ts-node src/test/migrations/reset.ts", "migration-steps:run": "ts-node src/test/migrations/", "migration-steps:watch": "ts-node-dev --respawn --transpile-only --exit-child --watch src/test/migrations/ src/test/migrations/ -- --watch", "run:dev": "run-p run:dev:client run:dev:server", diff --git a/src/test/migrations/create-migration-step.sh b/src/test/migrations/create-migration-step.sh deleted file mode 100755 index 4900f4a83f..0000000000 --- a/src/test/migrations/create-migration-step.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ] - then - echo "Give the name of the migration step as parameter" - exit 2 -fi - -. .env -_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -_DATE=$(date +%Y%m%d%H%M%S) -_FILENAME=$_DATE-step-$1.ts -_FILEPATH=$_DIR/steps/$_FILENAME - -cp "$_DIR/steps/template.ts" "$_FILEPATH" - -sed -i '' "s/#NAME#/$1/g" "$_FILEPATH" - -echo "Created migration step $_FILEPATH" diff --git a/src/test/migrations/createMigrationStep.ts b/src/test/migrations/createMigrationStep.ts new file mode 100644 index 0000000000..845b167de3 --- /dev/null +++ b/src/test/migrations/createMigrationStep.ts @@ -0,0 +1,66 @@ +import 'tsconfig-paths/register' +import 'dotenv/config' + +import * as fs from 'fs' +import * as path from 'path' + +import { Logger } from 'server/utils/logger' + +import { getUserInput } from './utils' + +/** + * Ask the user for the name of the migration step + * @returns Migration step name + */ +const askStepName = (): Promise => { + return getUserInput('Please enter a name for the migration step: ') +} + +/** + * Get the current date in the format YYYYMMDDHHMMSS + * @returns The current date in the format YYYYMMDDHHMMSS + */ +const getDate = (): string => { + const now = new Date() + return now + .toISOString() + .replace(/[-:T.]/g, '') + .slice(0, 14) +} + +export const createMigrationStep = async (initialStepName: string): Promise<{ fileName: string; filePath: string }> => { + let stepName = initialStepName + if (!stepName) { + stepName = await askStepName() + if (!stepName) { + Logger.error('Migration step name cannot be empty. Exiting.') + process.exit(1) + } + } + + const currentDir = __dirname + + const fileName = `${getDate()}-step-${stepName}.ts` + const filePath = path.join(currentDir, 'steps', fileName) + + const template = stepName === 'reset' ? 'template-reset.ts' : 'template.ts' + const templatePath = path.join(currentDir, 'steps', template) + + try { + let content = fs.readFileSync(templatePath, 'utf8') + content = content.replace(/#NAME#/g, stepName) + + fs.writeFileSync(filePath, content) + Logger.info(`Created migration step ${filePath}`) + } catch (error) { + Logger.error('Error creating migration step:', error) + process.exit(1) + } + + return { fileName, filePath } +} + +if (require.main === module) { + const stepName = process.argv[2] + createMigrationStep(stepName) +} diff --git a/src/test/migrations/index.ts b/src/test/migrations/index.ts index 3a55383cbc..44af0924f8 100644 --- a/src/test/migrations/index.ts +++ b/src/test/migrations/index.ts @@ -1,9 +1,6 @@ import 'tsconfig-paths/register' import 'dotenv/config' -import * as fs from 'fs' -import * as path from 'path' - import { VisitCycleLinksQueueFactory } from 'server/controller/cycleData/links/visitCycleLinks/queueFactory' import { WorkerFactory as VisitLinksWorkerFactory } from 'server/controller/cycleData/links/visitCycleLinks/workerFactory' import { UpdateDependenciesQueueFactory } from 'server/controller/cycleData/updateDependencies/queueFactory' @@ -12,6 +9,8 @@ import { DB } from 'server/db' import { RedisData } from 'server/repository/redis/redisData' import { Logger } from 'server/utils/logger' +import { getMigrationFiles } from './utils' + const client = DB let migrationSteps: Array let previousMigrations: Array = [] @@ -38,9 +37,7 @@ const tableDDL = ` const init = async () => { await client.query(tableDDL) previousMigrations = await client.map('select * from migrations.steps', [], (row) => row.name) - migrationSteps = fs - .readdirSync(path.join(__dirname, `steps`)) - .filter((file) => file !== 'template.ts' && file.endsWith('.ts') && !previousMigrations.includes(file)) + migrationSteps = getMigrationFiles(true).filter((file) => !previousMigrations.includes(file)) } const close = async () => { diff --git a/src/test/migrations/reset.ts b/src/test/migrations/reset.ts new file mode 100644 index 0000000000..69698019fa --- /dev/null +++ b/src/test/migrations/reset.ts @@ -0,0 +1,90 @@ +import 'tsconfig-paths/register' +import 'dotenv/config' + +import * as fs from 'fs' +import * as path from 'path' + +import { DB } from 'server/db' +import { Logger } from 'server/utils/logger' + +import { createMigrationStep } from './createMigrationStep' +import { getFilesToRemove, getMigrationFiles, getUserInput } from './utils' + +const _confirmReset = async (): Promise => { + const answer = await getUserInput('Type "reset" to proceed with migration-steps reset: ') + return answer.toLowerCase() === 'reset' +} + +const checkAllMigrationsRan = async () => { + const files = getMigrationFiles() + + const dbMigrations = await DB.manyOrNone('select * from migrations.steps;') + + const missingMigrations = files.filter((file) => !dbMigrations.some((dbMigration) => dbMigration.name === file)) + + if (missingMigrations.length > 0) { + Logger.warn('The following migrations have not been run:') + missingMigrations.forEach((file) => Logger.warn(`- ${file}`)) + return false + } + + return true +} + +const deleteOldMigrationFiles = (latestResetStep: string) => { + const stepsDir = path.join(__dirname, 'steps') + const filesToRemove = getFilesToRemove(latestResetStep) + + filesToRemove.forEach((file) => { + fs.unlinkSync(path.join(stepsDir, file)) + Logger.info(`Removed migration file: ${file}`) + }) +} + +const resetMigrationSteps = async () => { + try { + // === 1. Check if all migrations ran + const allMigrationsRan = await checkAllMigrationsRan() + if (!allMigrationsRan) { + Logger.error('Not all migrations have been run. Please run all migrations before resetting.') + return + } + + // === 2. Confirm reset from user + const confirmed = await _confirmReset() + if (!confirmed) { + Logger.info('Reset operation cancelled') + return + } + + // === 3. Create reset step + const { filePath, fileName } = await createMigrationStep('reset') + // eslint-disable-next-line @typescript-eslint/no-var-requires,global-require,import/no-dynamic-require + const resetStep = require(filePath).default + + await DB.tx(async (t) => { + // === 4. Run reset step + await resetStep(t, fileName) + + // === 5. Delete old migration files + deleteOldMigrationFiles(fileName) + }) + + Logger.info('Migration steps reset completed successfully') + } catch (error) { + Logger.error('Error resetting migration steps:', error) + throw error + } finally { + await DB.$pool.end() + } +} + +Logger.info('Starting migration steps reset') +resetMigrationSteps() + .then(() => { + Logger.info('Migration steps reset finished') + process.exit(0) + }) + .catch(() => { + process.exit(1) + }) diff --git a/src/test/migrations/steps/template-reset.ts b/src/test/migrations/steps/template-reset.ts new file mode 100644 index 0000000000..54a5a0f110 --- /dev/null +++ b/src/test/migrations/steps/template-reset.ts @@ -0,0 +1,19 @@ +import { BaseProtocol } from 'server/db' +import { Logger } from 'server/utils/logger' + +export default async (client: BaseProtocol, fileName: string) => { + try { + // === 1. Truncate migration_steps table and reset the primary key + await client.none('truncate table migrations.steps restart identity') + Logger.info('Truncated migrations.steps table and reset primary key') + + // === 2. Insert the current reset step into the migrations.steps table + await client.none('insert into migrations.steps (name) values ($1)', [fileName]) + Logger.info(`Inserted reset step ${fileName} into migrations.steps table`) + + Logger.info('Database table migrations.steps reset completed successfully') + } catch (error) { + Logger.error('Error resetting database:', error) + throw error + } +} diff --git a/src/test/migrations/utils/index.ts b/src/test/migrations/utils/index.ts new file mode 100644 index 0000000000..a640ded967 --- /dev/null +++ b/src/test/migrations/utils/index.ts @@ -0,0 +1,50 @@ +import * as fs from 'fs' +import * as path from 'path' +import * as readline from 'readline' + +/** + * Get user input from the terminal + * @param question - The question to ask the user + * @returns The user input + */ +export const getUserInput = (question: string): Promise => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }) + + return new Promise((resolve) => { + rl.question(question, (answer) => { + rl.close() + resolve(answer.trim()) + }) + }) +} + +/** + * Get the migration files from the migration steps directory + * @param includeResetSteps - Whether to include reset steps in the list + * @returns The migration files + */ +export const getMigrationFiles = (includeResetSteps = false): Array => { + const stepsDir = path.join(__dirname, '..', 'steps') + return fs.readdirSync(stepsDir).filter((file) => { + const isResetStep = file.includes('step-reset') + return ( + file.endsWith('.ts') && + file !== 'template.ts' && + file !== 'template-reset.ts' && + (includeResetSteps || !isResetStep) + ) + }) +} + +/** + * Get the files to remove from the migration steps directory + * @param latestResetStep - The name of the latest reset step + * @returns The files to remove, excluding the latest reset step + */ +export const getFilesToRemove = (latestResetStep: string): Array => { + const allFiles = getMigrationFiles(true) + return allFiles.filter((file) => file !== latestResetStep) +}