Upgrade Elasticsearch from 7.17 to 8

docker-compose.deps.yml

@@ -18,7 +18,7 @@ services:
     restart: unless-stopped
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.19
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3
     restart: unless-stopped
     environment:
       - discovery.type=single-node

docker-compose.dev-deps.yml

@@ -27,10 +27,11 @@ services:
       - './data/backups/elasticsearch:/data/backups/elasticsearch'
     environment:
       - 'discovery.type=single-node'
+      - 'cluster.routing.allocation.disk.watermark.enable_for_single_data_node=true'
       - 'cluster.routing.allocation.disk.threshold_enabled=false'
       - path.repo=/data/backups/elasticsearch
       - 'ES_JAVA_OPTS=-Xms1024m -Xmx1024m'
-      - bootstrap.system_call_filter=false
+      - xpack.security.enabled=false # elasticsearch >8.x defaults to true
       # - http.port=9200
       # - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358
       # - http.cors.enabled=true

docker-compose.yml

@@ -253,6 +253,7 @@ services:
       - HEARTH_MONGO_URL=mongodb://mongo1/hearth-dev
       - DASHBOARD_MONGO_URL=mongodb://mongo1/performance
       - OPENHIM_MONGO_URL=mongodb://mongo1/openhim-dev
+      - SEARCH_URL=http://search:9090/
       - ES_HOST=elasticsearch:9200
       - INFLUX_HOST=influxdb
       - INFLUX_PORT=8086

packages/commons/src/fhir/task.ts

@@ -197,12 +197,15 @@ export function isTaskOrTaskHistory<T extends Resource>(
 ): resource is (T & TaskHistory) | (T & Task) {
   return ['TaskHistory', 'Task'].includes(resource.resourceType)
 }
-export function getTaskFromSavedBundle<T extends SavedBundle>(bundle: T) {
+export function getTaskFromSavedBundle<T extends SavedBundle>(
+  bundle: T
+): SavedTask {
   const task = bundle.entry.map(({ resource }) => resource).find(isTask)
 
   if (!task || !isSaved(task)) {
     throw new Error('No task found in bundle')
   }
+
   return task
 }
 

packages/commons/src/index.ts

@@ -13,3 +13,4 @@ export * from './uuid'
 export * from './documents'
 export * from './http'
 export * from './logger'
+export * from './search'

packages/commons/src/search.ts

@@ -0,0 +1,143 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * OpenCRVS is also distributed under the terms of the Civil Registration
+ * & Healthcare Disclaimer located at http://opencrvs.org/license.
+ *
+ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+ */
+
+export const enum EVENT {
+  BIRTH = 'Birth',
+  DEATH = 'Death',
+  MARRIAGE = 'Marriage'
+}
+
+export const IN_PROGRESS_STATUS = 'IN_PROGRESS'
+export const ARCHIVED_STATUS = 'ARCHIVED'
+export const DECLARED_STATUS = 'DECLARED'
+export const REJECTED_STATUS = 'REJECTED'
+export const VALIDATED_STATUS = 'VALIDATED'
+const WAITING_VALIDATION_STATUS = 'WAITING_VALIDATION'
+export const REGISTERED_STATUS = 'REGISTERED'
+const REINSTATED_STATUS = 'REINSTATED'
+export const CERTIFIED_STATUS = 'CERTIFIED'
+export const ISSUED_STATUS = 'ISSUED'
+const REQUESTED_CORRECTION_STATUS = 'REQUESTED_CORRECTION'
+const DECLARATION_UPDATED_STATUS = 'DECLARATION_UPDATED'
+
+export const validStatusMapping = {
+  [ARCHIVED_STATUS]: [
+    DECLARED_STATUS,
+    REJECTED_STATUS,
+    VALIDATED_STATUS
+  ] as const,
+  [IN_PROGRESS_STATUS]: [null] as const,
+  [DECLARED_STATUS]: [ARCHIVED_STATUS, null] as const,
+  [REJECTED_STATUS]: [
+    DECLARED_STATUS,
+    IN_PROGRESS_STATUS,
+    WAITING_VALIDATION_STATUS,
+    VALIDATED_STATUS,
+    ARCHIVED_STATUS
+  ] as const,
+  [VALIDATED_STATUS]: [
+    DECLARED_STATUS,
+    IN_PROGRESS_STATUS,
+    REJECTED_STATUS,
+    ARCHIVED_STATUS,
+    DECLARATION_UPDATED_STATUS,
+    null
+  ] as const,
+  [WAITING_VALIDATION_STATUS]: [
+    null,
+    DECLARED_STATUS,
+    IN_PROGRESS_STATUS,
+    REJECTED_STATUS,
+    VALIDATED_STATUS,
+    DECLARATION_UPDATED_STATUS
+  ] as const,
+  [REGISTERED_STATUS]: [
+    null,
+    DECLARED_STATUS,
+    IN_PROGRESS_STATUS,
+    REJECTED_STATUS,
+    VALIDATED_STATUS,
+    WAITING_VALIDATION_STATUS
+  ] as const,
+  [CERTIFIED_STATUS]: [REGISTERED_STATUS, ISSUED_STATUS] as const,
+  [ISSUED_STATUS]: [CERTIFIED_STATUS] as const,
+  [REQUESTED_CORRECTION_STATUS]: [REGISTERED_STATUS, CERTIFIED_STATUS] as const,
+  [REINSTATED_STATUS]: [ARCHIVED_STATUS] as const
+}
+
+export interface ICorrection {
+  section: string
+  fieldName: string
+  oldValue: string
+  newValue: string | number | boolean
+}
+
+export interface IAssignment {
+  practitionerId: string
+  firstName: string
+  lastName: string
+  officeName: string
+}
+
+export interface IOperationHistory {
+  operationType: keyof typeof validStatusMapping
+  operatedOn: string
+}
+
+export interface SearchDocument {
+  compositionId: string
+  compositionType?: string
+  event?: EVENT
+  type?: string
+  informantType?: string
+  contactNumber?: string
+  contactEmail?: string
+  dateOfDeclaration?: string
+  trackingId?: string
+  registrationNumber?: string
+  eventLocationId?: string
+  eventJurisdictionIds?: string[]
+  eventCountry?: string
+  declarationLocationId?: string
+  declarationJurisdictionIds?: string[]
+  rejectReason?: string
+  rejectComment?: string
+  relatesTo?: string[]
+  childFirstNames?: string
+  childFamilyName?: string
+  childFirstNamesLocal?: string
+  motherFirstNames?: string
+  motherFamilyName?: string
+  motherDoB?: string
+  motherIdentifier?: string
+  childDoB?: string
+  childIdentifier?: string
+  createdBy?: string
+  updatedBy?: string
+  createdAt?: string
+  modifiedAt?: string
+  assignment?: IAssignment | null
+  operationHistories?: IOperationHistory[]
+}
+
+/**
+ * Takes up elastic search total and returns the number of documents independent of 'rest_total_hits_as_int' setting
+ * https://github.com/elastic/elasticsearch-js/issues/1604
+ * @param total total number of documents
+ */
+export const getSearchTotalCount = (
+  total: number | { value: number } | undefined
+) => {
+  if (typeof total === 'number') {
+    return total
+  }
+  return total?.value || 0
+}

packages/commons/src/types.ts

@@ -13,5 +13,6 @@ export * from './fhir/transformers/input'
 export * from './record'
 export * from './test-resources'
 export * from './nominal'
+export * from './search'
 
 export type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>

packages/gateway/package.json

@@ -17,11 +17,11 @@
     "gen:schema:watch": "nodemon --quiet --on-change-only -e graphql -i src/graphql/schema.graphql -x 'yarn gen:schema'",
     "gen:types:watch": "nodemon --quiet --on-change-only -w src/graphql/schema.graphql -x 'yarn gen:types'",
     "open:cov": "yarn test && opener coverage/index.html",
-    "lint": "eslint -c .eslintrc.js --fix ./src --max-warnings=0",
+    "lint": "eslint -c .eslintrc.js --fix ./src",
     "precommit": "lint-staged"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "7.17.13",
+    "@elastic/elasticsearch": "8.13.1",
     "@graphql-tools/graphql-file-loader": "^7.5.16",
     "@graphql-tools/load": "^7.8.13",
     "@graphql-tools/schema": "^9.0.17",
@@ -82,6 +82,7 @@
     "@types/jwt-decode": "^2.2.1",
     "@types/lodash": "^4.14.108",
     "@types/node-fetch": "^2.5.12",
+    "@types/node": "18.11.18",
     "@types/redis": "^2.8.6",
     "@types/uuid": "^3.4.3",
     "@typescript-eslint/eslint-plugin": "^4.5.0",

packages/gateway/src/features/search/root-resolvers.ts

@@ -8,7 +8,6 @@
  *
  * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
  */
-import { ApiResponse } from '@elastic/elasticsearch'
 import { getMetrics, postMetrics } from '@gateway/features/metrics/service'
 import {
   getSystem,
@@ -22,6 +21,11 @@ import { ISearchCriteria, postAdvancedSearch } from './utils'
 import { fetchRegistrationForDownloading } from '@gateway/workflow/index'
 import { ApolloError } from 'apollo-server-hapi'
 
+type ApiResponse<T> = {
+  body: T
+  statusCode: number
+}
+
 export class RateLimitError extends ApolloError {
   constructor(message: string) {
     super(message, 'DAILY_QUOTA_EXCEEDED')

packages/migration/package.json

@@ -18,13 +18,13 @@
     "status:openhim": "migrate-mongo status -f ./src/migrate-mongo-config-openhim.js",
     "status:user-mgnt": "migrate-mongo status -f ./src/migrate-mongo-config-user-mgnt.js",
     "status:application-config": "migrate-mongo status -f ./src/migrate-mongo-config-application-config.js",
-    "reindex-search": "tsx src/reindex-search.ts",
+    "reindex-search": "NODE_OPTIONS=--dns-result-order=ipv4first tsx src/reindex-search.ts",
     "precommit": "lint-staged",
     "test:compilation": "tsc --noEmit",
     "build": "rimraf build && tsc"
   },
   "dependencies": {
-    "@elastic/elasticsearch": "7.17.13",
+    "@elastic/elasticsearch": "8.13.1",
     "@types/bcryptjs": "^2.4.2",
     "@types/lodash-es": "^4.17.0",
     "@types/uuid": "^3.4.3",
@@ -52,6 +52,13 @@
       "prettier --write"
     ]
   },
+  "modulePaths": [
+    "<rootDir>"
+  ],
+  "moduleNameMapper": {
+    "@migration/(.*)": "<rootDir>/src/$1",
+    "@opencrvs/commons/(.*)": "@opencrvs/commons/build/dist/$1"
+  },
   "keywords": [
     "migration",
     "nodejs"

packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts

@@ -63,6 +63,7 @@ export const up = async (db: Db, client: MongoClient) => {
             const compositionId =
               taskDoc.focus?.reference?.replace('Composition/', '') || ''
             const searchResult = await searchByCompositionId(compositionId)
+
             const operationHistoriesData =
               searchResult &&
               searchResult.body.hits.hits.length > 0 &&

packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts

@@ -17,8 +17,8 @@ import { Db, MongoClient } from 'mongodb'
 import {
   updateComposition,
   searchCompositionByCriteria
-  // eslint-disable-next-line import/no-relative-parent-imports
 } from '../../utils/elasticsearch-helper.js'
+import { getSearchTotalCount } from '@opencrvs/commons'
 
 // THIS MIGRATION POPULATES THE MISSING EVENTLOCATIONIDS OR EVENTJURISDICTIONS IDS DUE TO THE ISSUE - #5242
 
@@ -56,8 +56,9 @@ export const up = async (db: Db, client: MongoClient) => {
 
   const compositionsWithoutLocationIdsResult =
     await searchCompositionByCriteria(searchCriteria)
-  const totalCompositionsWithoutLocationIds =
-    compositionsWithoutLocationIdsResult?.body.hits.total.value || 0
+  const totalCompositionsWithoutLocationIds = getSearchTotalCount(
+    compositionsWithoutLocationIdsResult?.body?.hits.total
+  )
 
   while (processedDocCount < totalCompositionsWithoutLocationIds) {
     const elasticDocBatchResult = await searchCompositionByCriteria(
@@ -79,7 +80,7 @@ export const up = async (db: Db, client: MongoClient) => {
       } catch (error: any) {
         // eslint-disable-next-line no-console
         console.error(
-          `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds for ${elasticDoc.id} failed : ${error.stack}`
+          `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds for ${elasticDoc._id} failed: ${error.stack}`
         )
       }
     }
@@ -90,7 +91,7 @@ export const up = async (db: Db, client: MongoClient) => {
   await session.endSession()
   // eslint-disable-next-line no-console
   console.log(
-    `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds  completed successfully.`
+    `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds completed successfully.`
   )
 }