index user if not existed and log last active time for user accessing apis via backend apps

package.json

@@ -15,7 +15,8 @@
     "seed": "cd test/rumors-db && npm run seed",
     "pretest": "npm run rumors-db:test",
     "test": "NODE_ENV=test ELASTICSEARCH_URL=http://localhost:62223 jest --runInBand",
-    "start": "pm2 startOrRestart ecosystem.config.js --env production --no-daemon",
+    "posttest": "NODE_ENV=test ELASTICSEARCH_URL=http://localhost:62223 babel-node test/postTest.js",
+    "start": "pm2 startOrRestart ecosystem.config.js --env production --no-daemon",    
     "lint": "eslint src/.",
     "lint:fix": "eslint --fix src/.",
     "rumors-db:pull": "cd test/rumors-db && git pull",
@@ -83,7 +84,8 @@
     ],
     "setupFilesAfterEnv": [
       "./test/setup.js"
-    ]
+    ],
+    "testSequencer": "./test/testSequencer.js"
   },
   "engines": {
     "node": ">=12"

src/__tests__/auth.js

@@ -7,8 +7,8 @@ import client from 'util/client';
 const FIXED_DATE = 612921600000;
 
 describe('verifyProfile', () => {
-  beforeAll(async () => loadFixtures(fixtures));
-  afterAll(async () => unloadFixtures(fixtures));
+  beforeAll(() => loadFixtures(fixtures));
+  afterAll(() => unloadFixtures(fixtures));
 
   it('authenticates user via profile ID', async () => {
     const passportProfile = {

src/graphql/dataLoaders/__tests__/analyticsLoaderFactory.js

@@ -7,7 +7,7 @@ const loader = new DataLoaders();
 MockDate.set(1578589200000); // 2020-01-10T01:00:00.000+08:00
 
 describe('analyticsLoaderFactory', () => {
-  beforeAll(async () => await loadFixtures(fixtures));
+  beforeAll(() => loadFixtures(fixtures));
 
   it('should load last 31 days of data for given id', async () => {
     const res = await loader.analyticsLoader.load({
@@ -64,5 +64,5 @@ describe('analyticsLoaderFactory', () => {
     expect(error).toBe('docType is required');
   });
 
-  afterAll(async () => await unloadFixtures(fixtures));
+  afterAll(() => unloadFixtures(fixtures));
 });

src/graphql/models/__tests__/User.js

@@ -15,8 +15,8 @@ jest.mock('lodash', () => ({
 }));
 
 describe('User model', () => {
-  beforeAll(async () => await loadFixtures(fixtures));
-  afterAll(async () => await unloadFixtures(fixtures));
+  beforeAll(() => loadFixtures(fixtures));
+  afterAll(() => unloadFixtures(fixtures));
 
   describe('currentUserOnlyField', () => {
     const user = {

src/graphql/mutations/__tests__/CreateOrUpdateUser.js

@@ -1,4 +1,4 @@
-import { loadFixtures, saveStateForIndices, clearIndices } from 'util/fixtures';
+import { loadFixtures, unloadFixtures } from 'util/fixtures';
 import client from 'util/client';
 import MockDate from 'mockdate';
 import fixtures from '../__fixtures__/CreateOrUpdateUser';
@@ -19,25 +19,10 @@ jest.mock('../../models/User', () => {
   };
 });
 
-jest.mock('../../../rollbarInstance', () => ({
-  __esModule: true,
-  default: { error: jest.fn() },
-}));
-
-let dbStates = {};
-
 describe('CreateOrUpdateUser', () => {
-  beforeAll(async () => {
-    dbStates = await saveStateForIndices(['users']);
-    await clearIndices(['users']);
-    await loadFixtures(fixtures);
-  });
+  beforeAll(() => loadFixtures(fixtures));
 
-  afterAll(async () => {
-    await clearIndices(['users']);
-    // restore db states to prevent affecting other tests
-    await loadFixtures(dbStates);
-  });
+  afterAll(() => unloadFixtures(fixtures));
 
   it('creates backend user if not existed', async () => {
     MockDate.set(1602288000000);
@@ -66,6 +51,7 @@ describe('CreateOrUpdateUser', () => {
     rollbar.error.mockClear();
 
     MockDate.reset();
+    await client.delete({ index: 'users', type: 'doc', id });
   });
 
   it("updates backend users' last active time if user already existed", async () => {

src/graphql/mutations/__tests__/CreateReply.js

@@ -135,7 +135,11 @@ describe('CreateReply', () => {
       type: 'doc',
       id: replyId,
     });
+
     expect(reply._source).toMatchSnapshot();
+
+    // Cleanup
+    await client.delete({ index: 'replies', type: 'doc', id: replyId });
   });
 
   it('should throw error since a reference is required for type !== NOT_ARTICLE', async () => {

src/graphql/queries/__tests__/ListReplies.js

@@ -1,17 +1,10 @@
-import { loadFixtures, clearIndices, saveStateForIndices } from 'util/fixtures';
+import { loadFixtures, unloadFixtures } from 'util/fixtures';
 import gql from 'util/GraphQL';
 import { getCursor } from 'graphql/util';
 import fixtures from '../__fixtures__/ListReplies';
 
-const indices = ['replies', 'urls'];
-let dbStates;
 describe('ListReplies', () => {
-  beforeAll(async () => {
-    // storing the current db states to restore to after the test is completed
-    dbStates = await saveStateForIndices(indices);
-    await clearIndices(indices);
-    await loadFixtures(fixtures);
-  });
+  beforeAll(() => loadFixtures(fixtures));
 
   it('lists all replies', async () => {
     expect(
@@ -390,9 +383,5 @@ describe('ListReplies', () => {
     ).toMatchSnapshot();
   });
 
-  afterAll(async () => {
-    await clearIndices(indices);
-    // restore db states to prevent affecting other tests
-    await loadFixtures(dbStates);
-  });
+  afterAll(() => unloadFixtures(fixtures));
 });

src/scripts/__tests__/fetchStatsFromGA.js

@@ -61,7 +61,7 @@ describe('fetchStatsFromGA', () => {
       yargs.argvMock.mockReset();
     });
 
-    it('without any arugments', async () => {
+    it('without any arguments', async () => {
       yargs.argvMock.mockReturnValue({
         useContentGroup: true,
         loadScript: false,
@@ -73,7 +73,7 @@ describe('fetchStatsFromGA', () => {
       expect(storeScriptInDBMock).not.toHaveBeenCalled();
     });
 
-    it('with date arugments', async () => {
+    it('with date arguments', async () => {
       yargs.argvMock.mockReturnValue({
         startDate: '2020-01-01',
         endDate: '2020-02-01',
@@ -93,7 +93,7 @@ describe('fetchStatsFromGA', () => {
       expect(storeScriptInDBMock).not.toHaveBeenCalled();
     });
 
-    it('with loadScript arugments', async () => {
+    it('with loadScript arguments', async () => {
       yargs.argvMock.mockReturnValue({
         loadScript: true,
         useContentGroup: true,
@@ -105,7 +105,7 @@ describe('fetchStatsFromGA', () => {
       expect(storeScriptInDBMock).toHaveBeenCalled();
     });
 
-    it('with loadScript and date arugments', async () => {
+    it('with loadScript and date arguments', async () => {
       yargs.argvMock.mockReturnValue({
         loadScript: true,
         startDate: '2020-01-01',
@@ -474,8 +474,8 @@ describe('fetchStatsFromGA', () => {
         upsertDocStatsSpy.mockClear();
       });
 
-      afterAll(async () =>
-        await client.delete_script({ id: fetchStatsFromGA.upsertScriptID }));
+      afterAll(() =>
+        client.delete_script({ id: fetchStatsFromGA.upsertScriptID }));
 
       it('should aggregate rows of data', async () => {
         await fetchStatsFromGA.processReport(

src/scripts/migrations/__tests__/createBackendUsers.js

@@ -1,4 +1,4 @@
-import { loadFixtures, clearIndices, saveStateForIndices } from 'util/fixtures';
+import { loadFixtures } from 'util/fixtures';
 import client from 'util/client';
 import CreateBackendUsers from '../createBackendUsers';
 import fixtures from '../__fixtures__/createBackendUsers';
@@ -39,12 +39,8 @@ const indices = [
   'analytics',
 ];
 
-let dbStates = {};
 describe('createBackendUsers', () => {
   beforeAll(async () => {
-    // storing the current db states to restore to after the test is completed
-    dbStates = await saveStateForIndices(indices);
-    await clearIndices(indices);
     await loadFixtures(fixtures.fixturesToLoad);
 
     await new CreateBackendUsers({
@@ -60,9 +56,17 @@ describe('createBackendUsers', () => {
   });
 
   afterAll(async () => {
-    await clearIndices(indices);
-    // restore db states to prevent affecting other tests
-    await loadFixtures(dbStates);
+    for (const index of indices) {
+      await client.deleteByQuery({
+        index,
+        body: {
+          query: {
+            match_all: {},
+          },
+        },
+        refresh: 'true',
+      });
+    }
   });
 
   for (const index of indices) {

test/postTest.js

@@ -0,0 +1,32 @@
+import main from 'scripts/cleanupUrls';
+import client from 'util/client';
+
+
+const checkDocs = async () => {
+
+  const {body: aliases} = await client.cat.aliases({
+    format: 'JSON'
+  })
+
+  const indices = aliases.map(i => i.alias);
+
+  const { body: { hits: { total, hits } } } = await client.search({
+    index: indices,
+    _source: 'false'
+  })
+
+  if (total > 0) {
+    console.log('\x1b[33m');
+    console.log('WARNING: test db is not cleaned up properly.');
+    const docs = hits.map(d => `/${d._index}/${d._type}/${d._id}`)
+    console.log(JSON.stringify(docs, null, 2));
+    console.log('\x1b[0m');
+
+    for (const d of hits) {
+      await client.delete({index: d._index, type: d._type, id: d._id})
+    }
+    process.exit(1) 
+  }    
+}
+
+checkDocs()

test/setup.js

@@ -1,7 +1,15 @@
 import 'dotenv/config';
+
 jest.mock(__dirname + '../../src/util/grpc');
+
+jest.mock(__dirname + '../../src/rollbarInstance', () => ({
+  __esModule: true,
+  default: { error: jest.fn() },
+}));
+
 jest.setTimeout(process.env.JEST_TIMEOUT || 5000);
 
+
 expect.extend({
   toBeNaN(received) {
     const pass = isNaN(received);

test/testSequencer.js

@@ -0,0 +1,13 @@
+const Sequencer = require('@jest/test-sequencer').default;
+
+class TestSequencer extends Sequencer {
+  sort(tests) {
+    // Test structure information
+    // https://github.com/facebook/jest/blob/6b8b1404a1d9254e7d5d90a8934087a9c9899dab/packages/jest-runner/src/types.ts#L17-L21
+    const copyTests = Array.from(tests);
+    return copyTests.sort((testA, testB) => (testA.path > testB.path ? 1 : -1));
+  }
+}
+
+
+module.exports = TestSequencer;

test/util/fixtures.js

@@ -60,36 +60,3 @@ export async function resetFrom(fixtureMap, key) {
     refresh: 'true',
   });
 }
-
-export async function saveStateForIndices(indices) {
-  let states = {}
-  for (const index of indices) {
-    const { body: { hits: { hits } } } = await client.search({
-      index,
-      body: {
-        query: {
-          match_all: {},
-        },
-      },
-      size: 10000
-    })
-    for (const doc of hits) {
-      states[`/${doc._index}/${doc._type}/${doc._id}`] = { ...doc._source };
-    }
-  }
-  return states
-}
-
-export async function clearIndices(indices) {
-  for (const index of indices) {
-    await client.deleteByQuery({
-      index,
-      body: {
-        query: {
-          match_all: {},
-        },
-      },
-      refresh: 'true'
-    })
-  }
-}