3931 - yarn migration-steps:reset (#3979)

* 3931 - yarn migration-steps:reset

* 3931 - createMigrationStep: use TypeScript

* 3931 - createMigrationStep: use TypeScript

* 3931 - migration-steps:reset: check all migrations have ran

* fix typo in comment

* 3931 - migration-step utils: fix steps path

* migration-steps create: simplify getDate

* 3931 - fix reading user input

* 3931 - execute createMigrationStep only when called from terminal

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

package.json

@@ -16,7 +16,8 @@
     "heroku-cleanup": "cd node_modules/bcrypt && node-pre-gyp install --fallback-to-build",
     "migration-public:create": "src/tools/migrations/public/create-migration-step.sh",
     "migration-public:run": "ts-node src/tools/migrations/public",
-    "migration-steps:create": "src/test/migrations/create-migration-step.sh",
+    "migration-steps:create": "ts-node src/test/migrations/createMigrationStep.ts",
+    "migration-steps:reset": "ts-node src/test/migrations/reset.ts",
     "migration-steps:run": "ts-node src/test/migrations/",
     "migration-steps:watch": "ts-node-dev --respawn --transpile-only --exit-child --watch src/test/migrations/ src/test/migrations/ -- --watch",
     "run:dev": "run-p run:dev:client run:dev:server",

src/test/migrations/createMigrationStep.ts

@@ -0,0 +1,66 @@
+import 'tsconfig-paths/register'
+import 'dotenv/config'
+
+import * as fs from 'fs'
+import * as path from 'path'
+
+import { Logger } from 'server/utils/logger'
+
+import { getUserInput } from './utils'
+
+/**
+ * Ask the user for the name of the migration step
+ * @returns Migration step name
+ */
+const askStepName = (): Promise<string> => {
+  return getUserInput('Please enter a name for the migration step: ')
+}
+
+/**
+ * Get the current date in the format YYYYMMDDHHMMSS
+ * @returns The current date in the format YYYYMMDDHHMMSS
+ */
+const getDate = (): string => {
+  const now = new Date()
+  return now
+    .toISOString()
+    .replace(/[-:T.]/g, '')
+    .slice(0, 14)
+}
+
+export const createMigrationStep = async (initialStepName: string): Promise<{ fileName: string; filePath: string }> => {
+  let stepName = initialStepName
+  if (!stepName) {
+    stepName = await askStepName()
+    if (!stepName) {
+      Logger.error('Migration step name cannot be empty. Exiting.')
+      process.exit(1)
+    }
+  }
+
+  const currentDir = __dirname
+
+  const fileName = `${getDate()}-step-${stepName}.ts`
+  const filePath = path.join(currentDir, 'steps', fileName)
+
+  const template = stepName === 'reset' ? 'template-reset.ts' : 'template.ts'
+  const templatePath = path.join(currentDir, 'steps', template)
+
+  try {
+    let content = fs.readFileSync(templatePath, 'utf8')
+    content = content.replace(/#NAME#/g, stepName)
+
+    fs.writeFileSync(filePath, content)
+    Logger.info(`Created migration step ${filePath}`)
+  } catch (error) {
+    Logger.error('Error creating migration step:', error)
+    process.exit(1)
+  }
+
+  return { fileName, filePath }
+}
+
+if (require.main === module) {
+  const stepName = process.argv[2]
+  createMigrationStep(stepName)
+}

src/test/migrations/index.ts

@@ -1,9 +1,6 @@
 import 'tsconfig-paths/register'
 import 'dotenv/config'
 
-import * as fs from 'fs'
-import * as path from 'path'
-
 import { VisitCycleLinksQueueFactory } from 'server/controller/cycleData/links/visitCycleLinks/queueFactory'
 import { WorkerFactory as VisitLinksWorkerFactory } from 'server/controller/cycleData/links/visitCycleLinks/workerFactory'
 import { UpdateDependenciesQueueFactory } from 'server/controller/cycleData/updateDependencies/queueFactory'
@@ -12,6 +9,8 @@ import { DB } from 'server/db'
 import { RedisData } from 'server/repository/redis/redisData'
 import { Logger } from 'server/utils/logger'
 
+import { getMigrationFiles } from './utils'
+
 const client = DB
 let migrationSteps: Array<string>
 let previousMigrations: Array<string> = []
@@ -38,9 +37,7 @@ const tableDDL = `
 const init = async () => {
   await client.query(tableDDL)
   previousMigrations = await client.map('select * from migrations.steps', [], (row) => row.name)
-  migrationSteps = fs
-    .readdirSync(path.join(__dirname, `steps`))
-    .filter((file) => file !== 'template.ts' && file.endsWith('.ts') && !previousMigrations.includes(file))
+  migrationSteps = getMigrationFiles(true).filter((file) => !previousMigrations.includes(file))
 }
 
 const close = async () => {

src/test/migrations/reset.ts

@@ -0,0 +1,90 @@
+import 'tsconfig-paths/register'
+import 'dotenv/config'
+
+import * as fs from 'fs'
+import * as path from 'path'
+
+import { DB } from 'server/db'
+import { Logger } from 'server/utils/logger'
+
+import { createMigrationStep } from './createMigrationStep'
+import { getFilesToRemove, getMigrationFiles, getUserInput } from './utils'
+
+const _confirmReset = async (): Promise<boolean> => {
+  const answer = await getUserInput('Type "reset" to proceed with migration-steps reset: ')
+  return answer.toLowerCase() === 'reset'
+}
+
+const checkAllMigrationsRan = async () => {
+  const files = getMigrationFiles()
+
+  const dbMigrations = await DB.manyOrNone('select * from migrations.steps;')
+
+  const missingMigrations = files.filter((file) => !dbMigrations.some((dbMigration) => dbMigration.name === file))
+
+  if (missingMigrations.length > 0) {
+    Logger.warn('The following migrations have not been run:')
+    missingMigrations.forEach((file) => Logger.warn(`- ${file}`))
+    return false
+  }
+
+  return true
+}
+
+const deleteOldMigrationFiles = (latestResetStep: string) => {
+  const stepsDir = path.join(__dirname, 'steps')
+  const filesToRemove = getFilesToRemove(latestResetStep)
+
+  filesToRemove.forEach((file) => {
+    fs.unlinkSync(path.join(stepsDir, file))
+    Logger.info(`Removed migration file: ${file}`)
+  })
+}
+
+const resetMigrationSteps = async () => {
+  try {
+    // === 1. Check if all migrations ran
+    const allMigrationsRan = await checkAllMigrationsRan()
+    if (!allMigrationsRan) {
+      Logger.error('Not all migrations have been run. Please run all migrations before resetting.')
+      return
+    }
+
+    // === 2. Confirm reset from user
+    const confirmed = await _confirmReset()
+    if (!confirmed) {
+      Logger.info('Reset operation cancelled')
+      return
+    }
+
+    // === 3. Create reset step
+    const { filePath, fileName } = await createMigrationStep('reset')
+    // eslint-disable-next-line @typescript-eslint/no-var-requires,global-require,import/no-dynamic-require
+    const resetStep = require(filePath).default
+
+    await DB.tx(async (t) => {
+      // === 4. Run reset step
+      await resetStep(t, fileName)
+
+      // === 5. Delete old migration files
+      deleteOldMigrationFiles(fileName)
+    })
+
+    Logger.info('Migration steps reset completed successfully')
+  } catch (error) {
+    Logger.error('Error resetting migration steps:', error)
+    throw error
+  } finally {
+    await DB.$pool.end()
+  }
+}
+
+Logger.info('Starting migration steps reset')
+resetMigrationSteps()
+  .then(() => {
+    Logger.info('Migration steps reset finished')
+    process.exit(0)
+  })
+  .catch(() => {
+    process.exit(1)
+  })

src/test/migrations/steps/template-reset.ts

@@ -0,0 +1,19 @@
+import { BaseProtocol } from 'server/db'
+import { Logger } from 'server/utils/logger'
+
+export default async (client: BaseProtocol, fileName: string) => {
+  try {
+    // === 1. Truncate migration_steps table and reset the primary key
+    await client.none('truncate table migrations.steps restart identity')
+    Logger.info('Truncated migrations.steps table and reset primary key')
+
+    // === 2. Insert the current reset step into the migrations.steps table
+    await client.none('insert into migrations.steps (name) values ($1)', [fileName])
+    Logger.info(`Inserted reset step ${fileName} into migrations.steps table`)
+
+    Logger.info('Database table migrations.steps reset completed successfully')
+  } catch (error) {
+    Logger.error('Error resetting database:', error)
+    throw error
+  }
+}

src/test/migrations/utils/index.ts

@@ -0,0 +1,50 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import * as readline from 'readline'
+
+/**
+ * Get user input from the terminal
+ * @param question - The question to ask the user
+ * @returns The user input
+ */
+export const getUserInput = (question: string): Promise<string> => {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  })
+
+  return new Promise((resolve) => {
+    rl.question(question, (answer) => {
+      rl.close()
+      resolve(answer.trim())
+    })
+  })
+}
+
+/**
+ * Get the migration files from the migration steps directory
+ * @param includeResetSteps - Whether to include reset steps in the list
+ * @returns The migration files
+ */
+export const getMigrationFiles = (includeResetSteps = false): Array<string> => {
+  const stepsDir = path.join(__dirname, '..', 'steps')
+  return fs.readdirSync(stepsDir).filter((file) => {
+    const isResetStep = file.includes('step-reset')
+    return (
+      file.endsWith('.ts') &&
+      file !== 'template.ts' &&
+      file !== 'template-reset.ts' &&
+      (includeResetSteps || !isResetStep)
+    )
+  })
+}
+
+/**
+ * Get the files to remove from the migration steps directory
+ * @param latestResetStep - The name of the latest reset step
+ * @returns The files to remove, excluding the latest reset step
+ */
+export const getFilesToRemove = (latestResetStep: string): Array<string> => {
+  const allFiles = getMigrationFiles(true)
+  return allFiles.filter((file) => file !== latestResetStep)
+}