From 78ce4870a4e4268a599b182d323319af67304b4d Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 12 Jul 2024 14:36:41 +0300 Subject: [PATCH 01/15] chore(upgrade-elastic): bump elastic version to 8.14.3 --- docker-compose.deps.yml | 2 +- docker-compose.dev-deps.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.deps.yml b/docker-compose.deps.yml index 9f8975da89..83d677c6b8 100644 --- a/docker-compose.deps.yml +++ b/docker-compose.deps.yml @@ -18,7 +18,7 @@ services: restart: unless-stopped elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.19 + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3 restart: unless-stopped environment: - discovery.type=single-node diff --git a/docker-compose.dev-deps.yml b/docker-compose.dev-deps.yml index 236d8372a0..f69e3f837c 100644 --- a/docker-compose.dev-deps.yml +++ b/docker-compose.dev-deps.yml @@ -27,6 +27,7 @@ services: - './data/backups/elasticsearch:/data/backups/elasticsearch' environment: - 'discovery.type=single-node' + - 'cluster.routing.allocation.disk.watermark.enable_for_single_data_node=true' - 'cluster.routing.allocation.disk.threshold_enabled=false' - path.repo=/data/backups/elasticsearch - 'ES_JAVA_OPTS=-Xms1024m -Xmx1024m' From e75647f1a0695efc5e55f3aa3078b4b97918da3a Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 12 Jul 2024 14:56:23 +0300 Subject: [PATCH 02/15] chore(upgrade-elastic): upgrade elastic npm packages --- packages/gateway/package.json | 2 +- packages/migration/package.json | 2 +- packages/search/package.json | 2 +- .../registration/deduplicate/test-util.ts | 15 +++-- yarn.lock | 57 +++++++++++-------- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/packages/gateway/package.json b/packages/gateway/package.json index 5c4f77324d..a56639c962 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -21,7 +21,7 @@ "precommit": "lint-staged" }, "dependencies": { - "@elastic/elasticsearch": "7.17.13", + "@elastic/elasticsearch": "8.14.0", "@graphql-tools/graphql-file-loader": "^7.5.16", "@graphql-tools/load": "^7.8.13", "@graphql-tools/schema": "^9.0.17", diff --git a/packages/migration/package.json b/packages/migration/package.json index 8e0f33f656..32725bcf5b 100644 --- a/packages/migration/package.json +++ b/packages/migration/package.json @@ -24,7 +24,7 @@ "build": "rimraf build && tsc" }, "dependencies": { - "@elastic/elasticsearch": "7.17.13", + "@elastic/elasticsearch": "8.14.0", "@types/bcryptjs": "^2.4.2", "@types/lodash-es": "^4.17.0", "@types/uuid": "^3.4.3", diff --git a/packages/search/package.json b/packages/search/package.json index c7a0749735..88ecac12b0 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -17,7 +17,7 @@ "build:clean": "rm -rf build" }, "dependencies": { - "@elastic/elasticsearch": "7.17.13", + "@elastic/elasticsearch": "8.14.0", "@hapi/boom": "^9.1.1", "@hapi/hapi": "^20.0.1", "@opencrvs/commons": "^1.3.0", diff --git a/packages/search/src/features/registration/deduplicate/test-util.ts b/packages/search/src/features/registration/deduplicate/test-util.ts index 0c375c919c..eb4c9bf489 100644 --- a/packages/search/src/features/registration/deduplicate/test-util.ts +++ b/packages/search/src/features/registration/deduplicate/test-util.ts @@ -20,16 +20,19 @@ import { searchForDeathDuplicates, searchForBirthDuplicates } from './service' const ELASTIC_SEARCH_HTTP_PORT = 9200 const container: ElasticsearchContainer = new ElasticsearchContainer( - 'elasticsearch:7.17.7' + 'elasticsearch:8.14.0' ) export const startContainer = async (): Promise => { - return container - .withExposedPorts(ELASTIC_SEARCH_HTTP_PORT) - .withStartupTimeout(120_000) - .withEnvironment({ 'discovery.type': 'single-node' }) - .start() + return ( + container + .withExposedPorts(ELASTIC_SEARCH_HTTP_PORT) + .withStartupTimeout(120_000) + // @todo: check if this is necessary + .withEnvironment({ 'discovery.type': 'single-node' }) + .start() + ) } export const stopContainer = async ( diff --git a/yarn.lock b/yarn.lock index 6e199d2ac4..17ca039e01 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2433,15 +2433,26 @@ dependencies: "@elastic/ecs-helpers" "^1.1.0" -"@elastic/elasticsearch@7.17.13": - version "7.17.13" - resolved "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.17.13.tgz" - integrity sha512-GMXtFVqd3FgUlTtPL/GDc+3GhwvfZ0kSuegCvVVqb58kd+0I6U6u7PL8QFRLHtwzqLEBmYLdwr4PRkBAWKGlzA== +"@elastic/elasticsearch@8.14.0": + version "8.14.0" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.14.0.tgz#93b1f2a7cb6cc5cd1ceebf5060576bc690432e0a" + integrity sha512-MGrgCI4y+Ozssf5Q2IkVJlqt5bUMnKIICG2qxeOfrJNrVugMCBCAQypyesmSSocAtNm8IX3LxfJ3jQlFHmKe2w== dependencies: - debug "^4.3.1" - hpagent "^0.1.1" + "@elastic/transport" "^8.6.0" + tslib "^2.4.0" + +"@elastic/transport@^8.6.0": + version "8.7.0" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.7.0.tgz#006987fc5583f61c266e0b1003371e82efc7a6b5" + integrity sha512-IqXT7a8DZPJtqP2qmX1I2QKmxYyN27kvSW4g6pInESE1SuGwZDp2FxHJ6W2kwmYOJwQdAt+2aWwzXO5jHo9l4A== + dependencies: + "@opentelemetry/api" "1.x" + debug "^4.3.4" + hpagent "^1.0.0" ms "^2.1.3" secure-json-parse "^2.4.0" + tslib "^2.4.0" + undici "^6.12.0" "@emotion/babel-utils@^0.6.4": version "0.6.10" @@ -4975,6 +4986,11 @@ resolved "https://registry.npmjs.org/@open-draft/until/-/until-1.0.3.tgz" integrity sha512-Aq58f5HiWdyDlFffbbSjAlv596h/cOnt2DO1w3DOC7OJ5EHs0hd/nycJfiu9RJbT6Yk6F1knnRRXNSpxoIVZ9Q== +"@opentelemetry/api@1.x": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@opentelemetry/api@^1.4.1": version "1.6.0" resolved "https://registry.npmjs.org/@opentelemetry/api/-/api-1.6.0.tgz" @@ -14848,11 +14864,16 @@ hosted-git-info@^7.0.0: dependencies: lru-cache "^10.0.1" -hpagent@^0.1.1, hpagent@^0.1.2: +hpagent@^0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/hpagent/-/hpagent-0.1.2.tgz" integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ== +hpagent@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-1.2.0.tgz#0ae417895430eb3770c03443456b8d90ca464903" + integrity sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA== + html-element-map@^1.2.0: version "1.3.1" resolved "https://registry.npmjs.org/html-element-map/-/html-element-map-1.3.1.tgz" @@ -22651,23 +22672,6 @@ string-width@^2.1.1: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" -string-width@^1.0.1: - version "1.0.2" - resolved "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz" - integrity sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw== - dependencies: - code-point-at "^1.0.0" - is-fullwidth-code-point "^1.0.0" - strip-ansi "^3.0.0" - -string-width@^2.1.1: - version "2.1.1" - resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz" - integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== - dependencies: - is-fullwidth-code-point "^2.0.0" - strip-ansi "^4.0.0" - string-width@^3.0.0, string-width@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz" @@ -24014,6 +24018,11 @@ undici-types@~5.26.4: resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^6.12.0: + version "6.19.2" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.19.2.tgz#231bc5de78d0dafb6260cf454b294576c2f3cd31" + integrity sha512-JfjKqIauur3Q6biAtHJ564e3bWa8VvT+7cSiOJHFbX4Erv6CLGDpg8z+Fmg/1OI/47RA+GI2QZaF48SSaLvyBA== + unicode-byte-truncate@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/unicode-byte-truncate/-/unicode-byte-truncate-1.0.0.tgz" From f21d7d525e375e1cd635af51a5327d514a2f3571 Mon Sep 17 00:00:00 2001 From: Markus Date: Fri, 12 Jul 2024 15:56:35 +0300 Subject: [PATCH 03/15] chore(elastic-upgrade): ignore types ignore types in order to test the setup --- packages/commons/src/fhir/task.ts | 5 +- .../src/features/search/root-resolvers.ts | 7 +- ...5851-change-certified-status-to-issused.ts | 1 + ...0230602120944-populate-valid-locationid.ts | 2 + .../src/utils/elasticsearch-helper.ts | 44 ++- packages/search/src/config/routes.ts | 5 +- packages/search/src/elasticsearch/dbhelper.ts | 20 +- packages/search/src/elasticsearch/utils.ts | 4 +- .../features/registration/birth/service.ts | 9 +- .../features/registration/death/service.ts | 2 +- .../registration/deduplicate/service.ts | 316 +++++++++--------- .../features/registration/marriage/service.ts | 2 +- packages/search/src/features/reindex/prune.ts | 32 +- .../search/src/features/reindex/reindex.ts | 16 +- .../search/src/features/search/handler.ts | 40 ++- .../search/src/features/search/service.ts | 5 +- 16 files changed, 282 insertions(+), 228 deletions(-) diff --git a/packages/commons/src/fhir/task.ts b/packages/commons/src/fhir/task.ts index 45bc2631d6..bb05c4b98d 100644 --- a/packages/commons/src/fhir/task.ts +++ b/packages/commons/src/fhir/task.ts @@ -197,12 +197,15 @@ export function isTaskOrTaskHistory( ): resource is (T & TaskHistory) | (T & Task) { return ['TaskHistory', 'Task'].includes(resource.resourceType) } -export function getTaskFromSavedBundle(bundle: T) { +export function getTaskFromSavedBundle( + bundle: T +): SavedTask { const task = bundle.entry.map(({ resource }) => resource).find(isTask) if (!task || !isSaved(task)) { throw new Error('No task found in bundle') } + return task } diff --git a/packages/gateway/src/features/search/root-resolvers.ts b/packages/gateway/src/features/search/root-resolvers.ts index c550363cb3..17c351cf93 100644 --- a/packages/gateway/src/features/search/root-resolvers.ts +++ b/packages/gateway/src/features/search/root-resolvers.ts @@ -8,7 +8,6 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { ApiResponse } from '@elastic/elasticsearch' import { getMetrics, postMetrics } from '@gateway/features/metrics/service' import { getSystem, @@ -22,6 +21,12 @@ import { ISearchCriteria, postAdvancedSearch } from './utils' import { fetchRegistrationForDownloading } from '@gateway/workflow/index' import { ApolloError } from 'apollo-server-hapi' +// transportresults +type ApiResponse = { + body: T + statusCode: number +} + export class RateLimitError extends ApolloError { constructor(message: string) { super(message, 'DAILY_QUOTA_EXCEEDED') diff --git a/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts b/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts index fe3949c70b..b685e4e7a8 100644 --- a/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts +++ b/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts @@ -66,6 +66,7 @@ export const up = async (db: Db, client: MongoClient) => { const operationHistoriesData = searchResult && searchResult.body.hits.hits.length > 0 && + // @ts-ignore searchResult.body.hits.hits[0]._source?.operationHistories const lastOperationHistory = operationHistoriesData && diff --git a/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts b/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts index 859e72aafc..55db84d824 100644 --- a/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts +++ b/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts @@ -57,6 +57,7 @@ export const up = async (db: Db, client: MongoClient) => { const compositionsWithoutLocationIdsResult = await searchCompositionByCriteria(searchCriteria) const totalCompositionsWithoutLocationIds = + // @ts-ignore compositionsWithoutLocationIdsResult?.body.hits.total.value || 0 while (processedDocCount < totalCompositionsWithoutLocationIds) { @@ -79,6 +80,7 @@ export const up = async (db: Db, client: MongoClient) => { } catch (error: any) { // eslint-disable-next-line no-console console.error( + // @ts-ignore `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds for ${elasticDoc.id} failed : ${error.stack}` ) } diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index 748e1cd78d..ae1d4353fe 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -27,7 +27,7 @@ export const updateComposition = async ( try { response = await client.update({ index: ELASTICSEARCH_INDEX_NAME, - type: 'compositions', + // type: 'compositions', @todo: check whether this should work id, body: { doc: body @@ -58,9 +58,7 @@ export const renameField = async ( } } }, - script: { - inline: `ctx._source.${newFieldName} = ctx._source.${oldFieldName}; ctx._source.remove("${oldFieldName}");` - } + script: `ctx._source.${newFieldName} = ctx._source.${oldFieldName}; ctx._source.remove("${oldFieldName}");` } }) return response @@ -72,16 +70,21 @@ export const renameField = async ( export const searchByCompositionId = async (compositionId: string) => { try { - return await client.search({ - index: ELASTICSEARCH_INDEX_NAME, - body: { - query: { - match: { - _id: compositionId + return await client.search( + { + index: ELASTICSEARCH_INDEX_NAME, + body: { + query: { + match: { + _id: compositionId + } } } + }, + { + meta: true } - }) + ) } catch (err) { console.error(`searchByCompositionId: error: ${err}`) return null @@ -93,14 +96,19 @@ export const searchCompositionByCriteria = async ( extraConfigs?: Record ) => { try { - return await client.search({ - index: ELASTICSEARCH_INDEX_NAME, - type: 'compositions', - body: { - query: criteriaObject, - ...extraConfigs + return await client.search( + { + index: ELASTICSEARCH_INDEX_NAME, + // type: 'compositions', @todo: check whether this should work + body: { + query: criteriaObject, + ...extraConfigs + } + }, + { + meta: true } - }) + ) } catch (err) { console.error(`searchCompositionByCriteria: error: ${err}`) return null diff --git a/packages/search/src/config/routes.ts b/packages/search/src/config/routes.ts index 045ad5842f..805f682276 100644 --- a/packages/search/src/config/routes.ts +++ b/packages/search/src/config/routes.ts @@ -62,10 +62,11 @@ export const getRoutes = () => { path: '/ping', handler: async (request: any, h: any) => { try { - const res = await client.ping() + const res = await client.ping(undefined, { meta: true }) + logger.info(res) return { - success: res.meta.connection.status === 'alive' + success: res?.meta?.connection?.status === 'alive' } } catch (error) { logger.error(error) diff --git a/packages/search/src/elasticsearch/dbhelper.ts b/packages/search/src/elasticsearch/dbhelper.ts index 6530686b15..e7d49a81d2 100644 --- a/packages/search/src/elasticsearch/dbhelper.ts +++ b/packages/search/src/elasticsearch/dbhelper.ts @@ -9,7 +9,6 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ import { OPENCRVS_INDEX_NAME } from '@search/constants' -import { ISearchResponse } from '@search/elasticsearch/client' import { SearchDocument } from '@search/elasticsearch/utils' import { logger } from '@opencrvs/commons' import * as elasticsearch from '@elastic/elasticsearch' @@ -60,16 +59,19 @@ export const searchByCompositionId = async ( client: elasticsearch.Client ) => { try { - return await client.search>({ - index: OPENCRVS_INDEX_NAME, - body: { - query: { - match: { - _id: compositionId + return await client.search( + { + index: OPENCRVS_INDEX_NAME, + body: { + query: { + match: { + _id: compositionId + } } } - } - }) + }, + { meta: true } + ) } catch (err) { logger.error(`searchByCompositionId: error: ${err}`) return null diff --git a/packages/search/src/elasticsearch/utils.ts b/packages/search/src/elasticsearch/utils.ts index cb8401347a..220e308d82 100644 --- a/packages/search/src/elasticsearch/utils.ts +++ b/packages/search/src/elasticsearch/utils.ts @@ -292,7 +292,9 @@ export async function detectDeathDuplicates( export async function getCreatedBy(compositionId: string) { const results = await searchByCompositionId(compositionId, client) - const result = results?.body?.hits?.hits[0]?._source as SearchDocument + + const result = results?.body?.hits?.hits[0] + ?._source as unknown as SearchDocument return result?.createdBy } diff --git a/packages/search/src/features/registration/birth/service.ts b/packages/search/src/features/registration/birth/service.ts index a65a1ec731..6d533e5c9e 100644 --- a/packages/search/src/features/registration/birth/service.ts +++ b/packages/search/src/features/registration/birth/service.ts @@ -66,7 +66,7 @@ export const composeDocument = ( record: SavedBundle, existingDocument?: Awaited> ) => { - const task = getTaskFromSavedBundle(record) + const task: any = getTaskFromSavedBundle(record) const composition = getComposition(record) const body: BirthDocument = { @@ -75,7 +75,8 @@ export const composeDocument = ( createdAt: (existingDocument && existingDocument.body.hits.hits.length > 0 && - existingDocument.body.hits.hits[0]._source.createdAt) || + // @ts-ignore + existingDocument.body.hits.hits[0]?._source!.createdAt) || Date.now().toString(), modifiedAt: Date.now().toString(), operationHistories: composeOperationHistories(record) as IOperationHistory[] @@ -121,7 +122,7 @@ function createIndexBody( createFatherIndex(body, composition, bundle) createInformantIndex(body, composition, bundle) createDeclarationIndex(body, composition, bundle) - const task = getTaskFromSavedBundle(bundle) + const task: any = getTaskFromSavedBundle(bundle) createStatusHistory(body, task) const assignment = findAssignment(bundle) @@ -253,7 +254,7 @@ function createDeclarationIndex( composition: SavedComposition, bundle: SavedBundle ) { - const task = getTaskFromSavedBundle(bundle) + const task: any = getTaskFromSavedBundle(bundle) const contactPersonExtention = findTaskExtension( task, 'http://opencrvs.org/specs/extension/contact-person' diff --git a/packages/search/src/features/registration/death/service.ts b/packages/search/src/features/registration/death/service.ts index 7907945d8d..37d39af22f 100644 --- a/packages/search/src/features/registration/death/service.ts +++ b/packages/search/src/features/registration/death/service.ts @@ -68,7 +68,7 @@ export const composeDocument = ( createdAt: (existingDocument && existingDocument.body.hits.hits.length > 0 && - existingDocument.body.hits.hits[0]._source.createdAt) || + existingDocument.body.hits.hits[0]?._source?.createdAt) || Date.now().toString(), operationHistories: composeOperationHistories(bundle) as IOperationHistory[] } diff --git a/packages/search/src/features/registration/deduplicate/service.ts b/packages/search/src/features/registration/deduplicate/service.ts index 7085b66dcd..b8b775be04 100644 --- a/packages/search/src/features/registration/deduplicate/service.ts +++ b/packages/search/src/features/registration/deduplicate/service.ts @@ -75,49 +75,52 @@ export const searchForBirthDuplicates = async ( ) { return [] } + + const motherMust: any = [ + // If mother identifier is provided, it needs to match 100% + body.motherIdentifier && { + match_phrase: { + motherIdentifier: body.motherIdentifier + } + }, + body.motherFirstNames && { + match: { + motherFirstNames: { + query: body.motherFirstNames, + fuzziness: FIRST_NAME_FUZZINESS + } + } + }, + body.motherFamilyName && { + match: { + motherFamilyName: { + query: body.motherFamilyName, + fuzziness: FIRST_NAME_FUZZINESS, + minimum_should_match: '100%' + } + } + }, + body.motherDoB && { + range: { + motherDoB: { + gte: subYears(new Date(body.motherDoB), 1).toISOString(), + lte: addYears(new Date(body.motherDoB), 1).toISOString() + } + } + }, + body.motherDoB && { + distance_feature: { + field: 'motherDoB', + pivot: '365d', + origin: new Date(body.motherDoB).toISOString(), + boost: 1.5 + } + } + ].filter(Boolean) + const mothersDetailsMatch = { bool: { - must: [ - // If mother identifier is provided, it needs to match 100% - body.motherIdentifier && { - match_phrase: { - motherIdentifier: body.motherIdentifier - } - }, - body.motherFirstNames && { - match: { - motherFirstNames: { - query: body.motherFirstNames, - fuzziness: FIRST_NAME_FUZZINESS - } - } - }, - body.motherFamilyName && { - match: { - motherFamilyName: { - query: body.motherFamilyName, - fuzziness: FIRST_NAME_FUZZINESS, - minimum_should_match: '100%' - } - } - }, - body.motherDoB && { - range: { - motherDoB: { - gte: subYears(new Date(body.motherDoB), 1).toISOString(), - lte: addYears(new Date(body.motherDoB), 1).toISOString() - } - } - }, - body.motherDoB && { - distance_feature: { - field: 'motherDoB', - pivot: '365d', - origin: new Date(body.motherDoB).toISOString(), - boost: 1.5 - } - } - ].filter(Boolean) + must: motherMust } } @@ -144,10 +147,60 @@ export const searchForBirthDuplicates = async ( } } + const must1: any = [ + body.childFirstNames && { + match: { + childFirstNames: { + query: body.childFirstNames, + fuzziness: FIRST_NAME_FUZZINESS + } + } + }, + body.childFamilyName && { + match: { + childFamilyName: { + query: body.childFamilyName, + fuzziness: FIRST_NAME_FUZZINESS, + minimum_should_match: '100%' + } + } + } + ].filter(Boolean) + + const bool1 = body.childDoB && { + bool: { + should: [ + { + bool: { + must: [ + { + range: { + childDoB: { + gte: subYears(new Date(body.childDoB), 3).toISOString(), + lte: addYears(new Date(body.childDoB), 3).toISOString() + } + } + }, + { + distance_feature: { + field: 'childDoB', + pivot: '365d', + origin: new Date(body.childDoB).toISOString(), + boost: 1 + } + } + ] + } + } + ] + } + } + + const must2: any = [bool1, mothersDetailsMatch].filter(Boolean) try { - const result = await client.search>({ - index: OPENCRVS_INDEX_NAME, - body: { + const result = await client.search( + { + index: OPENCRVS_INDEX_NAME, query: { bool: { should: [ @@ -161,73 +214,25 @@ export const searchForBirthDuplicates = async ( must: [ { bool: { - must: [ - body.childFirstNames && { - match: { - childFirstNames: { - query: body.childFirstNames, - fuzziness: FIRST_NAME_FUZZINESS - } - } - }, - body.childFamilyName && { - match: { - childFamilyName: { - query: body.childFamilyName, - fuzziness: FIRST_NAME_FUZZINESS, - minimum_should_match: '100%' - } - } - } - ].filter(Boolean) - } - }, - body.childDoB && { - bool: { - should: [ - { - bool: { - must: [ - { - range: { - childDoB: { - gte: subYears( - new Date(body.childDoB), - 3 - ).toISOString(), - lte: addYears( - new Date(body.childDoB), - 3 - ).toISOString() - } - } - }, - { - distance_feature: { - field: 'childDoB', - pivot: '365d', - origin: new Date( - body.childDoB - ).toISOString(), - boost: 1 - } - } - ] - } - } - ] + must: must1 } }, - mothersDetailsMatch + ...must2 ].filter(Boolean) } } ] } } + }, + { + meta: true } - }) - return result.body.hits.hits + ) + + return result.body.hits.hits as ISearchResponse< + BirthDocument | DeathDocument + >['hits']['hits'] } catch (err) { logger.error(`searchBirthDuplicates error: ${err}`) throw err @@ -247,60 +252,62 @@ export const searchForDeathDuplicates = async ( return [] } + const must1: any = [ + body.deceasedFirstNames && { + match: { + deceasedFirstNames: { + query: body.deceasedFirstNames, + fuzziness: FIRST_NAME_FUZZINESS + } + } + }, + body.deceasedFamilyName && { + match: { + deceasedFamilyName: { + query: body.deceasedFamilyName, + fuzziness: FIRST_NAME_FUZZINESS + } + } + }, + body.deceasedIdentifier && { + match_phrase: { + deceasedIdentifier: body.deceasedIdentifier + } + } + ].filter((clause) => !!clause) + + const must2 = body.deathDate + ? [ + { + range: { + deathDate: { + gte: subDays(new Date(body.deathDate), 5).toISOString(), + lte: addDays(new Date(body.deathDate), 5).toISOString() + } + } + }, + { + distance_feature: { + field: 'deathDate', + pivot: '5d', // 5 days + origin: new Date(body.deathDate).toISOString(), + boost: 1 + } + } + ] + : [] + try { - const result = await client.search>({ - index: OPENCRVS_INDEX_NAME, - body: { + const result = await client.search( + { + index: OPENCRVS_INDEX_NAME, query: { bool: { must: [ - body.deceasedFirstNames && { - match: { - deceasedFirstNames: { - query: body.deceasedFirstNames, - fuzziness: FIRST_NAME_FUZZINESS - } - } - }, - body.deceasedFamilyName && { - match: { - deceasedFamilyName: { - query: body.deceasedFamilyName, - fuzziness: FIRST_NAME_FUZZINESS - } - } - }, - body.deceasedIdentifier && { - match_phrase: { - deceasedIdentifier: body.deceasedIdentifier - } - }, + ...must1, { bool: { - must: [ - body.deathDate && { - range: { - deathDate: { - gte: subDays( - new Date(body.deathDate), - 5 - ).toISOString(), - lte: addDays( - new Date(body.deathDate), - 5 - ).toISOString() - } - } - }, - body.deathDate && { - distance_feature: { - field: 'deathDate', - pivot: '5d', // 5 days - origin: new Date(body.deathDate).toISOString(), - boost: 1 - } - } - ].filter(Boolean) + must: must2 } }, { @@ -334,9 +341,14 @@ export const searchForDeathDuplicates = async ( ].filter(Boolean) } } + }, + { + meta: true } - }) - return result.body.hits.hits + ) + return result.body.hits.hits as ISearchResponse< + BirthDocument | DeathDocument + >['hits']['hits'] } catch (err) { logger.error(`searchDeathDuplicates error: ${err}`) throw err diff --git a/packages/search/src/features/registration/marriage/service.ts b/packages/search/src/features/registration/marriage/service.ts index 0a86247478..61d7265218 100644 --- a/packages/search/src/features/registration/marriage/service.ts +++ b/packages/search/src/features/registration/marriage/service.ts @@ -68,7 +68,7 @@ export const composeDocument = ( createdAt: (existingDocument && existingDocument.body.hits.hits.length > 0 && - existingDocument.body.hits.hits[0]._source.createdAt) || + existingDocument.body.hits.hits[0]?._source?.createdAt) || Date.now().toString(), operationHistories: composeOperationHistories(bundle) as IOperationHistory[] } diff --git a/packages/search/src/features/reindex/prune.ts b/packages/search/src/features/reindex/prune.ts index 4d0d496caa..7f763bb95b 100644 --- a/packages/search/src/features/reindex/prune.ts +++ b/packages/search/src/features/reindex/prune.ts @@ -14,25 +14,31 @@ import { logger } from '@opencrvs/commons' /** Prunes all the indices that don't have an alias pointing to it */ export const prune = async () => { - const { body: indicesWithAlias } = await client.cat.aliases< - Array<{ index: `${typeof OPENCRVS_INDEX_NAME}-${string}` }> - >({ - format: 'json', - name: OPENCRVS_INDEX_NAME - }) - const { body: allIndices } = await client.cat.indices< - Array<{ index: `${typeof OPENCRVS_INDEX_NAME}-${string}` }> - >({ - format: 'json', - index: `${OPENCRVS_INDEX_NAME}-*` - }) + const { body: indicesWithAlias } = await client.cat.aliases( + { + format: 'json', + name: OPENCRVS_INDEX_NAME + }, + { + meta: true + } + ) + const { body: allIndices } = await client.cat.indices( + { + format: 'json', + index: `${OPENCRVS_INDEX_NAME}-*` + }, + { + meta: true + } + ) for (const { index } of allIndices) { const isAliasPointedToIndex = indicesWithAlias.some( ({ index: aliasIndex }) => aliasIndex === index ) - if (!isAliasPointedToIndex) { + if (!isAliasPointedToIndex && !!index) { logger.info(`Deleting index: ${index}`) await client.indices.delete({ index }) } diff --git a/packages/search/src/features/reindex/reindex.ts b/packages/search/src/features/reindex/reindex.ts index baddac1672..79515462c0 100644 --- a/packages/search/src/features/reindex/reindex.ts +++ b/packages/search/src/features/reindex/reindex.ts @@ -8,7 +8,6 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { OnDropDocument } from '@elastic/elasticsearch/lib/Helpers' import { EVENT_TYPE, ValidRecord } from '@opencrvs/commons/types' import { OPENCRVS_INDEX_NAME } from '@search/constants' import { client } from '@search/elasticsearch/client' @@ -69,7 +68,7 @@ export const reindex = async () => { _id: doc.compositionId } }), - onDrop(doc: OnDropDocument) { + onDrop(doc) { throw new Error( `Document ${doc.document.compositionId} couldn't be inserted` ) @@ -89,10 +88,15 @@ export const reindex = async () => { * Points the latest index (for example: ocrvs-20240523000000) - to an alias (example: ocrvs) */ export async function updateAliases() { - const { body: indices } = await client.cat.indices>({ - format: 'json', - index: `${OPENCRVS_INDEX_NAME}-*` - }) + const { body: indices } = await client.cat.indices( + { + format: 'json', + index: `${OPENCRVS_INDEX_NAME}-*` + }, + { + meta: true + } + ) const sortedIndices = orderBy(indices, 'index') const { index: latestIndex } = sortedIndices.at(-1)! diff --git a/packages/search/src/features/search/handler.ts b/packages/search/src/features/search/handler.ts index 4b28bf3b3e..ff51e07900 100644 --- a/packages/search/src/features/search/handler.ts +++ b/packages/search/src/features/search/handler.ts @@ -72,10 +72,12 @@ export async function getAllDocumentsHandler( } }, { + meta: true, ignore: [404] } ) - const count: number = allDocumentsCountCheck.body.hits.total.value + // @ts-ignore + const count: number = allDocumentsCountCheck?.body?.hits?.total?.value if (count > 5000) { return internal( 'Elastic contains over 5000 results. It is risky to return all without pagination.' @@ -133,8 +135,9 @@ export async function getStatusWiseRegistrationCountHandler( }) } - const response = await client.search<{ - aggregations?: { + const response = await client.search< + any, + { statusCounts: { buckets: Array<{ key: string @@ -142,23 +145,28 @@ export async function getStatusWiseRegistrationCountHandler( }> } } - }>({ - body: { - size: 0, - query: { - bool: { - must: matchRules - } - }, - aggs: { - statusCounts: { - terms: { - field: 'type.keyword' + >( + { + body: { + size: 0, + query: { + bool: { + must: matchRules + } + }, + aggs: { + statusCounts: { + terms: { + field: 'type.keyword' + } } } } + }, + { + meta: true } - }) + ) if (!response.body.aggregations) { return payload.status.map((status) => ({ diff --git a/packages/search/src/features/search/service.ts b/packages/search/src/features/search/service.ts index 767481983d..8f792b6d60 100644 --- a/packages/search/src/features/search/service.ts +++ b/packages/search/src/features/search/service.ts @@ -8,8 +8,7 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { client, ISearchResponse } from '@search/elasticsearch/client' -import { ApiResponse } from '@elastic/elasticsearch' +import { client } from '@search/elasticsearch/client' import { ISearchCriteria, SortOrder } from '@search/features/search/types' import { advancedQueryBuilder } from '@search/features/search/utils' import { logger } from '@opencrvs/commons' @@ -60,7 +59,7 @@ export const advancedSearch = async ( payload: ISearchCriteria ) => { const formattedParams = await formatSearchParams(payload, isExternalSearch) - let response: ApiResponse> + let response try { response = await client.search(formattedParams, { ignore: !isExternalSearch ? [404] : undefined From a83860f9cc1f4fbe18cc1ef6d7edc8a675daeb28 Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 15 Jul 2024 12:36:42 +0300 Subject: [PATCH 04/15] chore(elastic-upgrade): use 8.14.3, add missing environment variables elasticsearch 8 defaults to security.enabled. Fix tests to match new payload --- docker-compose.dev-deps.yml | 2 +- packages/gateway/package.json | 3 +- packages/migration/package.json | 2 +- .../src/utils/elasticsearch-helper.ts | 5 +- packages/search/package.json | 3 +- packages/search/src/elasticsearch/client.ts | 3 +- .../search/src/elasticsearch/dbhelper.test.ts | 38 ++++++++------- packages/search/src/elasticsearch/dbhelper.ts | 36 +++++++++------ .../features/registration/birth/service.ts | 6 +-- .../deduplicate/deduplicate.test.ts | 4 +- .../registration/deduplicate/test-util.ts | 18 ++++---- yarn.lock | 46 +++++++++++-------- 12 files changed, 96 insertions(+), 70 deletions(-) diff --git a/docker-compose.dev-deps.yml b/docker-compose.dev-deps.yml index f69e3f837c..6f1ba0436f 100644 --- a/docker-compose.dev-deps.yml +++ b/docker-compose.dev-deps.yml @@ -31,7 +31,7 @@ services: - 'cluster.routing.allocation.disk.threshold_enabled=false' - path.repo=/data/backups/elasticsearch - 'ES_JAVA_OPTS=-Xms1024m -Xmx1024m' - - bootstrap.system_call_filter=false + - xpack.security.enabled=false # elasticsearch >8.x defaults to true # - http.port=9200 # - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358 # - http.cors.enabled=true diff --git a/packages/gateway/package.json b/packages/gateway/package.json index a56639c962..063a74a728 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -21,7 +21,7 @@ "precommit": "lint-staged" }, "dependencies": { - "@elastic/elasticsearch": "8.14.0", + "@elastic/elasticsearch": "8.13.1", "@graphql-tools/graphql-file-loader": "^7.5.16", "@graphql-tools/load": "^7.8.13", "@graphql-tools/schema": "^9.0.17", @@ -82,6 +82,7 @@ "@types/jwt-decode": "^2.2.1", "@types/lodash": "^4.14.108", "@types/node-fetch": "^2.5.12", + "@types/node": "18.11.18", "@types/redis": "^2.8.6", "@types/uuid": "^3.4.3", "@typescript-eslint/eslint-plugin": "^4.5.0", diff --git a/packages/migration/package.json b/packages/migration/package.json index 32725bcf5b..83e5db8f22 100644 --- a/packages/migration/package.json +++ b/packages/migration/package.json @@ -24,7 +24,7 @@ "build": "rimraf build && tsc" }, "dependencies": { - "@elastic/elasticsearch": "8.14.0", + "@elastic/elasticsearch": "8.13.1", "@types/bcryptjs": "^2.4.2", "@types/lodash-es": "^4.17.0", "@types/uuid": "^3.4.3", diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index ae1d4353fe..55ef8008f1 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -9,13 +9,14 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { Client } from '@elastic/elasticsearch' +import { Client, HttpConnection } from '@elastic/elasticsearch' const ES_HOST = process.env.ES_HOST || 'localhost:9200' const ELASTICSEARCH_INDEX_NAME = 'ocrvs' export const client = new Client({ - node: `http://${ES_HOST}` + node: `http://${ES_HOST}`, + Connection: HttpConnection }) export const updateComposition = async ( diff --git a/packages/search/package.json b/packages/search/package.json index 88ecac12b0..9d8847a8cb 100644 --- a/packages/search/package.json +++ b/packages/search/package.json @@ -17,7 +17,7 @@ "build:clean": "rm -rf build" }, "dependencies": { - "@elastic/elasticsearch": "8.14.0", + "@elastic/elasticsearch": "8.13.1", "@hapi/boom": "^9.1.1", "@hapi/hapi": "^20.0.1", "@opencrvs/commons": "^1.3.0", @@ -52,6 +52,7 @@ "@types/jwt-decode": "^2.2.1", "@types/lodash": "^4.14.149", "@types/node-fetch": "^2.5.12", + "@types/node": "18.11.18", "@types/uuid": "^3.4.4", "cross-env": "^7.0.0", "jest": "27.5.1", diff --git a/packages/search/src/elasticsearch/client.ts b/packages/search/src/elasticsearch/client.ts index 4ae3feefee..efbc8c1982 100644 --- a/packages/search/src/elasticsearch/client.ts +++ b/packages/search/src/elasticsearch/client.ts @@ -51,5 +51,6 @@ export interface ISearchResponse { } export const client = new elasticsearch.Client({ - node: `http://${ES_HOST}` + node: `http://${ES_HOST}`, + Connection: elasticsearch.HttpConnection }) diff --git a/packages/search/src/elasticsearch/dbhelper.test.ts b/packages/search/src/elasticsearch/dbhelper.test.ts index bb3c9d2466..95f8613e6a 100644 --- a/packages/search/src/elasticsearch/dbhelper.test.ts +++ b/packages/search/src/elasticsearch/dbhelper.test.ts @@ -34,12 +34,15 @@ describe('elasticsearch db helper', () => { indexComposition('testId', mockCompositionBody, client) expect(indexSpy).toBeCalled() - expect(indexSpy).toBeCalledWith({ - body: mockCompositionBody, - id: 'testId', - index: 'ocrvs', - refresh: 'wait_for' - }) + expect(indexSpy).toBeCalledWith( + { + body: mockCompositionBody, + id: 'testId', + index: 'ocrvs', + refresh: 'wait_for' + }, + { meta: true } + ) }) it('should update a composition with proper configuration', async () => { @@ -50,25 +53,28 @@ describe('elasticsearch db helper', () => { updateSpy = jest.spyOn(client, 'update') updateComposition('testId', body, client) expect(updateSpy).toBeCalled() - expect(updateSpy).toBeCalledWith({ - index: 'ocrvs', - id: 'testId', - body: { - doc: body + expect(updateSpy).toBeCalledWith( + { + index: 'ocrvs', + id: 'testId', + body: { + doc: body + }, + refresh: 'wait_for' }, - refresh: 'wait_for' - }) + { meta: true } + ) }) it('should perform search for composition', async () => { searchSpy = jest.spyOn(client, 'search') - searchForBirthDuplicates(mockCompositionBody, client) + await searchForBirthDuplicates(mockCompositionBody, client) if ( searchSpy.mock && searchSpy.mock.calls[0] && searchSpy.mock.calls[0][0] ) { - expect(searchSpy.mock.calls[0][0].body.query).toBeDefined() + expect(searchSpy.mock.calls[0][0].query).toBeDefined() } else { throw new Error('Failed') } @@ -77,7 +83,7 @@ describe('elasticsearch db helper', () => { it('should perform search by composition id', async () => { searchByCompositionId('r1324-sd6k2-12121-1212', client) - expect(searchSpy.mock.calls[0][0].body.query).toBeDefined() + expect(searchSpy.mock.calls[0][0].query).toBeDefined() expect(searchSpy).toBeCalled() }) }) diff --git a/packages/search/src/elasticsearch/dbhelper.ts b/packages/search/src/elasticsearch/dbhelper.ts index e7d49a81d2..9d68c68454 100644 --- a/packages/search/src/elasticsearch/dbhelper.ts +++ b/packages/search/src/elasticsearch/dbhelper.ts @@ -20,12 +20,17 @@ export const indexComposition = async ( ) => { let response: any try { - response = await client.index({ - index: OPENCRVS_INDEX_NAME, - id: compositionIdentifier, - body, - refresh: 'wait_for' // makes the call wait until the change is available via search - }) + response = await client.index( + { + index: OPENCRVS_INDEX_NAME, + id: compositionIdentifier, + body, + refresh: 'wait_for' // makes the call wait until the change is available via search + }, + { + meta: true + } + ) } catch (e) { logger.error(`indexComposition: error: ${e}`) } @@ -39,14 +44,19 @@ export const updateComposition = async ( ) => { let response: any try { - response = await client.update({ - index: OPENCRVS_INDEX_NAME, - id, - body: { - doc: body + response = await client.update( + { + index: OPENCRVS_INDEX_NAME, + id, + body: { + doc: body + }, + refresh: 'wait_for' // makes the call wait until the change is available via search }, - refresh: 'wait_for' // makes the call wait until the change is available via search - }) + { + meta: true + } + ) } catch (e) { logger.error(`updateComposition: error: ${e}`) } diff --git a/packages/search/src/features/registration/birth/service.ts b/packages/search/src/features/registration/birth/service.ts index 6d533e5c9e..b68333e1b5 100644 --- a/packages/search/src/features/registration/birth/service.ts +++ b/packages/search/src/features/registration/birth/service.ts @@ -66,7 +66,7 @@ export const composeDocument = ( record: SavedBundle, existingDocument?: Awaited> ) => { - const task: any = getTaskFromSavedBundle(record) + const task = getTaskFromSavedBundle(record) const composition = getComposition(record) const body: BirthDocument = { @@ -122,7 +122,7 @@ function createIndexBody( createFatherIndex(body, composition, bundle) createInformantIndex(body, composition, bundle) createDeclarationIndex(body, composition, bundle) - const task: any = getTaskFromSavedBundle(bundle) + const task = getTaskFromSavedBundle(bundle) createStatusHistory(body, task) const assignment = findAssignment(bundle) @@ -254,7 +254,7 @@ function createDeclarationIndex( composition: SavedComposition, bundle: SavedBundle ) { - const task: any = getTaskFromSavedBundle(bundle) + const task = getTaskFromSavedBundle(bundle) const contactPersonExtention = findTaskExtension( task, 'http://opencrvs.org/specs/extension/contact-person' diff --git a/packages/search/src/features/registration/deduplicate/deduplicate.test.ts b/packages/search/src/features/registration/deduplicate/deduplicate.test.ts index 36e4ea8420..962fa804eb 100644 --- a/packages/search/src/features/registration/deduplicate/deduplicate.test.ts +++ b/packages/search/src/features/registration/deduplicate/deduplicate.test.ts @@ -54,7 +54,7 @@ afterAll(async () => { describe('deduplication tests', () => { it('checks elasticsearch is up', async () => { - await client.ping() + await client.ping(undefined, { meta: true }) }) describe('standard check', () => { @@ -224,7 +224,7 @@ describe('deduplication tests', () => { describe('deduplication tests for death', () => { it('death:checks elasticsearch is up', async () => { - await client.ping() + await client.ping(undefined, { meta: true }) }) describe('standard check for death duplication', () => { diff --git a/packages/search/src/features/registration/deduplicate/test-util.ts b/packages/search/src/features/registration/deduplicate/test-util.ts index eb4c9bf489..23f0afbe71 100644 --- a/packages/search/src/features/registration/deduplicate/test-util.ts +++ b/packages/search/src/features/registration/deduplicate/test-util.ts @@ -20,19 +20,19 @@ import { searchForDeathDuplicates, searchForBirthDuplicates } from './service' const ELASTIC_SEARCH_HTTP_PORT = 9200 const container: ElasticsearchContainer = new ElasticsearchContainer( - 'elasticsearch:8.14.0' + 'elasticsearch:8.14.3' ) export const startContainer = async (): Promise => { - return ( - container - .withExposedPorts(ELASTIC_SEARCH_HTTP_PORT) - .withStartupTimeout(120_000) - // @todo: check if this is necessary - .withEnvironment({ 'discovery.type': 'single-node' }) - .start() - ) + return container + .withExposedPorts(ELASTIC_SEARCH_HTTP_PORT) + .withStartupTimeout(120_000) + .withEnvironment({ + 'discovery.type': 'single-node', + 'xpack.security.enabled': 'false' + }) + .start() } export const stopContainer = async ( diff --git a/yarn.lock b/yarn.lock index 17ca039e01..7b0c6fd5e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2433,26 +2433,25 @@ dependencies: "@elastic/ecs-helpers" "^1.1.0" -"@elastic/elasticsearch@8.14.0": - version "8.14.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.14.0.tgz#93b1f2a7cb6cc5cd1ceebf5060576bc690432e0a" - integrity sha512-MGrgCI4y+Ozssf5Q2IkVJlqt5bUMnKIICG2qxeOfrJNrVugMCBCAQypyesmSSocAtNm8IX3LxfJ3jQlFHmKe2w== +"@elastic/elasticsearch@8.13.1": + version "8.13.1" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.13.1.tgz#0fbe8318cf7f21c599165bb901277428639d57ec" + integrity sha512-2G4Vu6OHw4+XTrp7AGIcOEezpPEoVrWg2JTK1v/exEKSLYquZkUdd+m4yOL3/UZ6bTj7hmXwrmYzW76BnLCkJQ== dependencies: - "@elastic/transport" "^8.6.0" + "@elastic/transport" "~8.4.1" tslib "^2.4.0" -"@elastic/transport@^8.6.0": - version "8.7.0" - resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.7.0.tgz#006987fc5583f61c266e0b1003371e82efc7a6b5" - integrity sha512-IqXT7a8DZPJtqP2qmX1I2QKmxYyN27kvSW4g6pInESE1SuGwZDp2FxHJ6W2kwmYOJwQdAt+2aWwzXO5jHo9l4A== +"@elastic/transport@~8.4.1": + version "8.4.1" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.4.1.tgz#f98c5a5e2156bcb3f01170b4aca7e7de4d8b61b8" + integrity sha512-/SXVuVnuU5b4dq8OFY4izG+dmGla185PcoqgK6+AJMpmOeY1QYVNbWtCwvSvoAANN5D/wV+EBU8+x7Vf9EphbA== dependencies: - "@opentelemetry/api" "1.x" debug "^4.3.4" hpagent "^1.0.0" ms "^2.1.3" secure-json-parse "^2.4.0" tslib "^2.4.0" - undici "^6.12.0" + undici "^5.22.1" "@emotion/babel-utils@^0.6.4": version "0.6.10" @@ -2808,6 +2807,11 @@ lodash.isundefined "^3.0.1" lodash.uniq "^4.5.0" +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@floating-ui/core@^1.4.2": version "1.5.0" resolved "https://registry.npmjs.org/@floating-ui/core/-/core-1.5.0.tgz" @@ -4986,11 +4990,6 @@ resolved "https://registry.npmjs.org/@open-draft/until/-/until-1.0.3.tgz" integrity sha512-Aq58f5HiWdyDlFffbbSjAlv596h/cOnt2DO1w3DOC7OJ5EHs0hd/nycJfiu9RJbT6Yk6F1knnRRXNSpxoIVZ9Q== -"@opentelemetry/api@1.x": - version "1.9.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" - integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== - "@opentelemetry/api@^1.4.1": version "1.6.0" resolved "https://registry.npmjs.org/@opentelemetry/api/-/api-1.6.0.tgz" @@ -7713,6 +7712,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190" integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw== +"@types/node@18.11.18": + version "18.11.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" + integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== + "@types/node@^10.1.0", "@types/node@^10.12.5": version "10.17.60" resolved "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz" @@ -24018,10 +24022,12 @@ undici-types@~5.26.4: resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== -undici@^6.12.0: - version "6.19.2" - resolved "https://registry.yarnpkg.com/undici/-/undici-6.19.2.tgz#231bc5de78d0dafb6260cf454b294576c2f3cd31" - integrity sha512-JfjKqIauur3Q6biAtHJ564e3bWa8VvT+7cSiOJHFbX4Erv6CLGDpg8z+Fmg/1OI/47RA+GI2QZaF48SSaLvyBA== +undici@^5.22.1: + version "5.28.4" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" + integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + dependencies: + "@fastify/busboy" "^2.0.0" unicode-byte-truncate@^1.0.0: version "1.0.0" From 14b67389ff62a09e60c5a7ba3417ea3be5f3eb9e Mon Sep 17 00:00:00 2001 From: Markus Date: Mon, 15 Jul 2024 17:26:50 +0300 Subject: [PATCH 05/15] chore(elastic-upgrade): remove explicit connection property forced production to use http --- packages/migration/src/utils/elasticsearch-helper.ts | 5 ++--- packages/search/src/elasticsearch/client.ts | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index 55ef8008f1..ae1d4353fe 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -9,14 +9,13 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { Client, HttpConnection } from '@elastic/elasticsearch' +import { Client } from '@elastic/elasticsearch' const ES_HOST = process.env.ES_HOST || 'localhost:9200' const ELASTICSEARCH_INDEX_NAME = 'ocrvs' export const client = new Client({ - node: `http://${ES_HOST}`, - Connection: HttpConnection + node: `http://${ES_HOST}` }) export const updateComposition = async ( diff --git a/packages/search/src/elasticsearch/client.ts b/packages/search/src/elasticsearch/client.ts index efbc8c1982..4ae3feefee 100644 --- a/packages/search/src/elasticsearch/client.ts +++ b/packages/search/src/elasticsearch/client.ts @@ -51,6 +51,5 @@ export interface ISearchResponse { } export const client = new elasticsearch.Client({ - node: `http://${ES_HOST}`, - Connection: elasticsearch.HttpConnection + node: `http://${ES_HOST}` }) From 6caf8b53e9f06c236589627eee23b2b8d0ca751a Mon Sep 17 00:00:00 2001 From: Markus Date: Tue, 16 Jul 2024 09:17:58 +0300 Subject: [PATCH 06/15] chore(elastic-upgrade): add NODE_OPTIONS ipv4first for reindexing Results to Error: connect ECONNREFUSED ::1:9090 otherwise on QA and locally --- packages/migration/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/migration/package.json b/packages/migration/package.json index 83e5db8f22..ae19da011c 100644 --- a/packages/migration/package.json +++ b/packages/migration/package.json @@ -18,7 +18,7 @@ "status:openhim": "migrate-mongo status -f ./src/migrate-mongo-config-openhim.js", "status:user-mgnt": "migrate-mongo status -f ./src/migrate-mongo-config-user-mgnt.js", "status:application-config": "migrate-mongo status -f ./src/migrate-mongo-config-application-config.js", - "reindex-search": "tsx src/reindex-search.ts", + "reindex-search": "cross-env NODE_OPTIONS=--dns-result-order=ipv4first tsx src/reindex-search.ts", "precommit": "lint-staged", "test:compilation": "tsc --noEmit", "build": "rimraf build && tsc" From 932ecb652ecf995db6c5341777c61950f5e87b62 Mon Sep 17 00:00:00 2001 From: Markus Date: Tue, 16 Jul 2024 09:29:27 +0300 Subject: [PATCH 07/15] chore(elastic-upgrade): ensure meta key is present on all elastic client calls --- .../src/utils/elasticsearch-helper.ts | 50 +++++++----- .../search/src/features/delete/handler.ts | 9 ++- packages/search/src/features/reindex/prune.ts | 2 +- .../search/src/features/reindex/reindex.ts | 81 +++++++++++-------- .../search/src/features/search/handler.ts | 3 +- .../search/src/features/search/service.ts | 3 +- 6 files changed, 87 insertions(+), 61 deletions(-) diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index ae1d4353fe..35c9b3b5b6 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -25,15 +25,20 @@ export const updateComposition = async ( ) => { let response try { - response = await client.update({ - index: ELASTICSEARCH_INDEX_NAME, - // type: 'compositions', @todo: check whether this should work - id, - body: { - doc: body + response = await client.update( + { + index: ELASTICSEARCH_INDEX_NAME, + // type: 'compositions', @todo: check whether this should work + id, + body: { + doc: body + }, + ...extraConfigs }, - ...extraConfigs - }) + { + meta: true + } + ) } catch (e) { console.error(`updateComposition: error: ${e}`) } @@ -46,21 +51,24 @@ export const renameField = async ( newFieldName: string ) => { try { - const response = await client.updateByQuery({ - index: ELASTICSEARCH_INDEX_NAME, - body: { - query: { - bool: { - must_not: { - exists: { - field: newFieldName + const response = await client.updateByQuery( + { + index: ELASTICSEARCH_INDEX_NAME, + body: { + query: { + bool: { + must_not: { + exists: { + field: newFieldName + } } } - } - }, - script: `ctx._source.${newFieldName} = ctx._source.${oldFieldName}; ctx._source.remove("${oldFieldName}");` - } - }) + }, + script: `ctx._source.${newFieldName} = ctx._source.${oldFieldName}; ctx._source.remove("${oldFieldName}");` + } + }, + { meta: true } + ) return response } catch (err) { console.error(`searchByCompositionId: error: ${err}`) diff --git a/packages/search/src/features/delete/handler.ts b/packages/search/src/features/delete/handler.ts index c53c90146d..f9e216b74a 100644 --- a/packages/search/src/features/delete/handler.ts +++ b/packages/search/src/features/delete/handler.ts @@ -20,9 +20,12 @@ export async function deleteOCRVSIndexHandler( ) { let response: any try { - response = await client.indices.delete({ - index: OPENCRVS_INDEX_NAME - }) + response = await client.indices.delete( + { + index: OPENCRVS_INDEX_NAME + }, + { meta: true } + ) logger.info(`Successfully deleted ${OPENCRVS_INDEX_NAME} index`) return h.response(response).code(200) } catch (err) { diff --git a/packages/search/src/features/reindex/prune.ts b/packages/search/src/features/reindex/prune.ts index 7f763bb95b..9f317bbbc4 100644 --- a/packages/search/src/features/reindex/prune.ts +++ b/packages/search/src/features/reindex/prune.ts @@ -40,7 +40,7 @@ export const prune = async () => { if (!isAliasPointedToIndex && !!index) { logger.info(`Deleting index: ${index}`) - await client.indices.delete({ index }) + await client.indices.delete({ index }, { meta: true }) } } } diff --git a/packages/search/src/features/reindex/reindex.ts b/packages/search/src/features/reindex/reindex.ts index 79515462c0..3ad6141242 100644 --- a/packages/search/src/features/reindex/reindex.ts +++ b/packages/search/src/features/reindex/reindex.ts @@ -48,32 +48,42 @@ export const reindex = async () => { } }) - await client.indices.create({ - index, - body: { - settings: { - number_of_shards: 1, - number_of_replicas: 0 + await client.indices.create( + { + index, + body: { + settings: { + number_of_shards: 1, + number_of_replicas: 0 + } } + }, + { + meta: true } - }) + ) - await client.helpers.bulk({ - retries: 3, - wait: 3000, - datasource: stream.pipe(transformedStreamData), - onDocument: (doc: BirthDocument) => ({ - index: { - _index: index, - _id: doc.compositionId + await client.helpers.bulk( + { + retries: 3, + wait: 3000, + datasource: stream.pipe(transformedStreamData), + onDocument: (doc: BirthDocument) => ({ + index: { + _index: index, + _id: doc.compositionId + } + }), + onDrop(doc) { + throw new Error( + `Document ${doc.document.compositionId} couldn't be inserted` + ) } - }), - onDrop(doc) { - throw new Error( - `Document ${doc.document.compositionId} couldn't be inserted` - ) + }, + { + meta: true } - }) + ) const t2 = performance.now() logger.info( `Finished reindexing to ${index} in ${((t2 - t1) / 1000).toFixed( @@ -101,17 +111,20 @@ export async function updateAliases() { const sortedIndices = orderBy(indices, 'index') const { index: latestIndex } = sortedIndices.at(-1)! - await client.indices.updateAliases({ - body: { - actions: [ - { - remove: { - alias: OPENCRVS_INDEX_NAME, - index: `${OPENCRVS_INDEX_NAME}-*` - } - }, - { add: { alias: OPENCRVS_INDEX_NAME, index: latestIndex } } - ] - } - }) + await client.indices.updateAliases( + { + body: { + actions: [ + { + remove: { + alias: OPENCRVS_INDEX_NAME, + index: `${OPENCRVS_INDEX_NAME}-*` + } + }, + { add: { alias: OPENCRVS_INDEX_NAME, index: latestIndex } } + ] + } + }, + { meta: true } + ) } diff --git a/packages/search/src/features/search/handler.ts b/packages/search/src/features/search/handler.ts index ff51e07900..c894a0967b 100644 --- a/packages/search/src/features/search/handler.ts +++ b/packages/search/src/features/search/handler.ts @@ -94,7 +94,8 @@ export async function getAllDocumentsHandler( } }, { - ignore: [404] + ignore: [404], + meta: true } ) return h.response(allDocuments).code(200) diff --git a/packages/search/src/features/search/service.ts b/packages/search/src/features/search/service.ts index 8f792b6d60..39ae68bcb2 100644 --- a/packages/search/src/features/search/service.ts +++ b/packages/search/src/features/search/service.ts @@ -62,7 +62,8 @@ export const advancedSearch = async ( let response try { response = await client.search(formattedParams, { - ignore: !isExternalSearch ? [404] : undefined + ignore: !isExternalSearch ? [404] : undefined, + meta: true }) } catch (error) { if (error.statusCode === 400) { From 406e7465b3f010407a8329ff6e62d7df23728aaf Mon Sep 17 00:00:00 2001 From: Markus Date: Tue, 16 Jul 2024 09:54:39 +0300 Subject: [PATCH 08/15] chore(elastic-upgrade): remove cross-env reference from reindex package script --- packages/migration/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/migration/package.json b/packages/migration/package.json index ae19da011c..a965d327ad 100644 --- a/packages/migration/package.json +++ b/packages/migration/package.json @@ -18,7 +18,7 @@ "status:openhim": "migrate-mongo status -f ./src/migrate-mongo-config-openhim.js", "status:user-mgnt": "migrate-mongo status -f ./src/migrate-mongo-config-user-mgnt.js", "status:application-config": "migrate-mongo status -f ./src/migrate-mongo-config-application-config.js", - "reindex-search": "cross-env NODE_OPTIONS=--dns-result-order=ipv4first tsx src/reindex-search.ts", + "reindex-search": "NODE_OPTIONS=--dns-result-order=ipv4first tsx src/reindex-search.ts", "precommit": "lint-staged", "test:compilation": "tsc --noEmit", "build": "rimraf build && tsc" From 5600f938f16018d6d35d554697c1a484de67b3f1 Mon Sep 17 00:00:00 2001 From: Markus Date: Tue, 16 Jul 2024 10:36:56 +0300 Subject: [PATCH 09/15] chore(elastic-upgrade): add search url to migration docker envs --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index d9eb4b3fc9..97518b2d18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -253,6 +253,7 @@ services: - HEARTH_MONGO_URL=mongodb://mongo1/hearth-dev - DASHBOARD_MONGO_URL=mongodb://mongo1/performance - OPENHIM_MONGO_URL=mongodb://mongo1/openhim-dev + - SEARCH_URL=http://search:9090/ - ES_HOST=elasticsearch:9200 - INFLUX_HOST=influxdb - INFLUX_PORT=8086 From 5bf1d00c3bad62bed4283af4fafe5ff04a2502c1 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 17 Jul 2024 10:58:49 +0300 Subject: [PATCH 10/15] chore(elastic-upgrade): reuse search types between services --- packages/commons/src/index.ts | 1 + packages/commons/src/search.ts | 143 ++++++++++++++++++ packages/commons/src/types.ts | 1 + packages/gateway/package.json | 2 +- .../src/features/search/root-resolvers.ts | 1 - packages/migration/package.json | 7 + ...5851-change-certified-status-to-issused.ts | 2 +- ...0230602120944-populate-valid-locationid.ts | 13 +- .../src/utils/elasticsearch-helper.ts | 11 +- packages/migration/tsconfig.json | 5 +- packages/search/src/elasticsearch/dbhelper.ts | 8 +- packages/search/src/elasticsearch/utils.ts | 114 +------------- .../features/registration/birth/service.ts | 11 +- .../registration/deduplicate/service.ts | 18 +-- .../search/src/features/search/handler.ts | 13 +- 15 files changed, 200 insertions(+), 150 deletions(-) create mode 100644 packages/commons/src/search.ts diff --git a/packages/commons/src/index.ts b/packages/commons/src/index.ts index 491c5b6a8b..4ef19fe852 100644 --- a/packages/commons/src/index.ts +++ b/packages/commons/src/index.ts @@ -13,3 +13,4 @@ export * from './uuid' export * from './documents' export * from './http' export * from './logger' +export * from './search' diff --git a/packages/commons/src/search.ts b/packages/commons/src/search.ts new file mode 100644 index 0000000000..472a8147ea --- /dev/null +++ b/packages/commons/src/search.ts @@ -0,0 +1,143 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * OpenCRVS is also distributed under the terms of the Civil Registration + * & Healthcare Disclaimer located at http://opencrvs.org/license. + * + * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. + */ + +export const enum EVENT { + BIRTH = 'Birth', + DEATH = 'Death', + MARRIAGE = 'Marriage' +} + +export const IN_PROGRESS_STATUS = 'IN_PROGRESS' +export const ARCHIVED_STATUS = 'ARCHIVED' +export const DECLARED_STATUS = 'DECLARED' +export const REJECTED_STATUS = 'REJECTED' +export const VALIDATED_STATUS = 'VALIDATED' +const WAITING_VALIDATION_STATUS = 'WAITING_VALIDATION' +export const REGISTERED_STATUS = 'REGISTERED' +const REINSTATED_STATUS = 'REINSTATED' +export const CERTIFIED_STATUS = 'CERTIFIED' +export const ISSUED_STATUS = 'ISSUED' +const REQUESTED_CORRECTION_STATUS = 'REQUESTED_CORRECTION' +const DECLARATION_UPDATED_STATUS = 'DECLARATION_UPDATED' + +export const validStatusMapping = { + [ARCHIVED_STATUS]: [ + DECLARED_STATUS, + REJECTED_STATUS, + VALIDATED_STATUS + ] as const, + [IN_PROGRESS_STATUS]: [null] as const, + [DECLARED_STATUS]: [ARCHIVED_STATUS, null] as const, + [REJECTED_STATUS]: [ + DECLARED_STATUS, + IN_PROGRESS_STATUS, + WAITING_VALIDATION_STATUS, + VALIDATED_STATUS, + ARCHIVED_STATUS + ] as const, + [VALIDATED_STATUS]: [ + DECLARED_STATUS, + IN_PROGRESS_STATUS, + REJECTED_STATUS, + ARCHIVED_STATUS, + DECLARATION_UPDATED_STATUS, + null + ] as const, + [WAITING_VALIDATION_STATUS]: [ + null, + DECLARED_STATUS, + IN_PROGRESS_STATUS, + REJECTED_STATUS, + VALIDATED_STATUS, + DECLARATION_UPDATED_STATUS + ] as const, + [REGISTERED_STATUS]: [ + null, + DECLARED_STATUS, + IN_PROGRESS_STATUS, + REJECTED_STATUS, + VALIDATED_STATUS, + WAITING_VALIDATION_STATUS + ] as const, + [CERTIFIED_STATUS]: [REGISTERED_STATUS, ISSUED_STATUS] as const, + [ISSUED_STATUS]: [CERTIFIED_STATUS] as const, + [REQUESTED_CORRECTION_STATUS]: [REGISTERED_STATUS, CERTIFIED_STATUS] as const, + [REINSTATED_STATUS]: [ARCHIVED_STATUS] as const +} + +export interface ICorrection { + section: string + fieldName: string + oldValue: string + newValue: string | number | boolean +} + +export interface IAssignment { + practitionerId: string + firstName: string + lastName: string + officeName: string +} + +export interface IOperationHistory { + operationType: keyof typeof validStatusMapping + operatedOn: string +} + +export interface SearchDocument { + compositionId: string + compositionType?: string + event?: EVENT + type?: string + informantType?: string + contactNumber?: string + contactEmail?: string + dateOfDeclaration?: string + trackingId?: string + registrationNumber?: string + eventLocationId?: string + eventJurisdictionIds?: string[] + eventCountry?: string + declarationLocationId?: string + declarationJurisdictionIds?: string[] + rejectReason?: string + rejectComment?: string + relatesTo?: string[] + childFirstNames?: string + childFamilyName?: string + childFirstNamesLocal?: string + motherFirstNames?: string + motherFamilyName?: string + motherDoB?: string + motherIdentifier?: string + childDoB?: string + childIdentifier?: string + createdBy?: string + updatedBy?: string + createdAt?: string + modifiedAt?: string + assignment?: IAssignment | null + operationHistories?: IOperationHistory[] +} + +/** + * Takes up elastic search total and returns the number of documents independent of 'rest_total_hits_as_int' setting + * https://github.com/elastic/elasticsearch-js/issues/1604 + * @param total total number of documents + */ +export const getSearchTotalCount = ( + total: number | { value: number } | undefined +) => { + if (typeof total === 'number') { + return total + } + return total?.value || 0 +} diff --git a/packages/commons/src/types.ts b/packages/commons/src/types.ts index b761d4370c..530eb9871e 100644 --- a/packages/commons/src/types.ts +++ b/packages/commons/src/types.ts @@ -13,5 +13,6 @@ export * from './fhir/transformers/input' export * from './record' export * from './test-resources' export * from './nominal' +export * from './search' export type PartialBy = Omit & Partial> diff --git a/packages/gateway/package.json b/packages/gateway/package.json index 063a74a728..8b6beb847b 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -17,7 +17,7 @@ "gen:schema:watch": "nodemon --quiet --on-change-only -e graphql -i src/graphql/schema.graphql -x 'yarn gen:schema'", "gen:types:watch": "nodemon --quiet --on-change-only -w src/graphql/schema.graphql -x 'yarn gen:types'", "open:cov": "yarn test && opener coverage/index.html", - "lint": "eslint -c .eslintrc.js --fix ./src --max-warnings=0", + "lint": "eslint -c .eslintrc.js --fix ./src", "precommit": "lint-staged" }, "dependencies": { diff --git a/packages/gateway/src/features/search/root-resolvers.ts b/packages/gateway/src/features/search/root-resolvers.ts index 17c351cf93..7a7e46f499 100644 --- a/packages/gateway/src/features/search/root-resolvers.ts +++ b/packages/gateway/src/features/search/root-resolvers.ts @@ -21,7 +21,6 @@ import { ISearchCriteria, postAdvancedSearch } from './utils' import { fetchRegistrationForDownloading } from '@gateway/workflow/index' import { ApolloError } from 'apollo-server-hapi' -// transportresults type ApiResponse = { body: T statusCode: number diff --git a/packages/migration/package.json b/packages/migration/package.json index a965d327ad..5113f51a97 100644 --- a/packages/migration/package.json +++ b/packages/migration/package.json @@ -52,6 +52,13 @@ "prettier --write" ] }, + "modulePaths": [ + "" + ], + "moduleNameMapper": { + "@migration/(.*)": "/src/$1", + "@opencrvs/commons/(.*)": "@opencrvs/commons/build/dist/$1" + }, "keywords": [ "migration", "nodejs" diff --git a/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts b/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts index b685e4e7a8..1b9b6863a7 100644 --- a/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts +++ b/packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts @@ -63,10 +63,10 @@ export const up = async (db: Db, client: MongoClient) => { const compositionId = taskDoc.focus?.reference?.replace('Composition/', '') || '' const searchResult = await searchByCompositionId(compositionId) + const operationHistoriesData = searchResult && searchResult.body.hits.hits.length > 0 && - // @ts-ignore searchResult.body.hits.hits[0]._source?.operationHistories const lastOperationHistory = operationHistoriesData && diff --git a/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts b/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts index 55db84d824..446d6bcf84 100644 --- a/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts +++ b/packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts @@ -17,8 +17,8 @@ import { Db, MongoClient } from 'mongodb' import { updateComposition, searchCompositionByCriteria - // eslint-disable-next-line import/no-relative-parent-imports } from '../../utils/elasticsearch-helper.js' +import { getSearchTotalCount } from '@opencrvs/commons' // THIS MIGRATION POPULATES THE MISSING EVENTLOCATIONIDS OR EVENTJURISDICTIONS IDS DUE TO THE ISSUE - #5242 @@ -56,9 +56,9 @@ export const up = async (db: Db, client: MongoClient) => { const compositionsWithoutLocationIdsResult = await searchCompositionByCriteria(searchCriteria) - const totalCompositionsWithoutLocationIds = - // @ts-ignore - compositionsWithoutLocationIdsResult?.body.hits.total.value || 0 + const totalCompositionsWithoutLocationIds = getSearchTotalCount( + compositionsWithoutLocationIdsResult?.body?.hits.total + ) while (processedDocCount < totalCompositionsWithoutLocationIds) { const elasticDocBatchResult = await searchCompositionByCriteria( @@ -80,8 +80,7 @@ export const up = async (db: Db, client: MongoClient) => { } catch (error: any) { // eslint-disable-next-line no-console console.error( - // @ts-ignore - `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds for ${elasticDoc.id} failed : ${error.stack}` + `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds for ${elasticDoc._id} failed: ${error.stack}` ) } } @@ -92,7 +91,7 @@ export const up = async (db: Db, client: MongoClient) => { await session.endSession() // eslint-disable-next-line no-console console.log( - `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds completed successfully.` + `Migration - ElasticSearch :: Process for populating missing eventLocationId/eventJurisdictionIds completed successfully.` ) } diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index 35c9b3b5b6..bd8d5f7ff1 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -9,7 +9,9 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ +// @TODO: Unify this file with @search/src/elasticsearch/dbhelper.ts import { Client } from '@elastic/elasticsearch' +import { SearchDocument } from '@opencrvs/commons/types' const ES_HOST = process.env.ES_HOST || 'localhost:9200' const ELASTICSEARCH_INDEX_NAME = 'ocrvs' @@ -23,9 +25,8 @@ export const updateComposition = async ( body: any, extraConfigs?: Record ) => { - let response try { - response = await client.update( + return await client.update( { index: ELASTICSEARCH_INDEX_NAME, // type: 'compositions', @todo: check whether this should work @@ -43,7 +44,7 @@ export const updateComposition = async ( console.error(`updateComposition: error: ${e}`) } - return response + return } export const renameField = async ( @@ -78,7 +79,7 @@ export const renameField = async ( export const searchByCompositionId = async (compositionId: string) => { try { - return await client.search( + return await client.search( { index: ELASTICSEARCH_INDEX_NAME, body: { @@ -104,7 +105,7 @@ export const searchCompositionByCriteria = async ( extraConfigs?: Record ) => { try { - return await client.search( + return await client.search( { index: ELASTICSEARCH_INDEX_NAME, // type: 'compositions', @todo: check whether this should work diff --git a/packages/migration/tsconfig.json b/packages/migration/tsconfig.json index da242ccbfa..ec636545bd 100644 --- a/packages/migration/tsconfig.json +++ b/packages/migration/tsconfig.json @@ -1,8 +1,11 @@ { "compilerOptions": { "baseUrl": "./src", + "paths": { + "@migration/*": ["./*"] + }, "allowJs": true, - "moduleResolution": "node", + "moduleResolution": "node16", "target": "esnext", "module": "esnext", "esModuleInterop": true, diff --git a/packages/search/src/elasticsearch/dbhelper.ts b/packages/search/src/elasticsearch/dbhelper.ts index 9d68c68454..f5af6d5c57 100644 --- a/packages/search/src/elasticsearch/dbhelper.ts +++ b/packages/search/src/elasticsearch/dbhelper.ts @@ -9,8 +9,7 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ import { OPENCRVS_INDEX_NAME } from '@search/constants' -import { SearchDocument } from '@search/elasticsearch/utils' -import { logger } from '@opencrvs/commons' +import { logger, SearchDocument } from '@opencrvs/commons' import * as elasticsearch from '@elastic/elasticsearch' export const indexComposition = async ( @@ -42,9 +41,8 @@ export const updateComposition = async ( body: SearchDocument, client: elasticsearch.Client ) => { - let response: any try { - response = await client.update( + return await client.update( { index: OPENCRVS_INDEX_NAME, id, @@ -61,7 +59,7 @@ export const updateComposition = async ( logger.error(`updateComposition: error: ${e}`) } - return response + return } export const searchByCompositionId = async ( diff --git a/packages/search/src/elasticsearch/utils.ts b/packages/search/src/elasticsearch/utils.ts index 220e308d82..9753c5bc39 100644 --- a/packages/search/src/elasticsearch/utils.ts +++ b/packages/search/src/elasticsearch/utils.ts @@ -23,77 +23,16 @@ import { SavedBundle, SavedOffice, SavedPractitioner, - SavedTask + SavedTask, + SearchDocument, + validStatusMapping, + IOperationHistory } from '@opencrvs/commons/types' import { findName } from '@search/features/fhir/fhir-utils' -export const enum EVENT { - BIRTH = 'Birth', - DEATH = 'Death', - MARRIAGE = 'Marriage' -} - -export const IN_PROGRESS_STATUS = 'IN_PROGRESS' -export const ARCHIVED_STATUS = 'ARCHIVED' -export const DECLARED_STATUS = 'DECLARED' -export const REJECTED_STATUS = 'REJECTED' -export const VALIDATED_STATUS = 'VALIDATED' -const WAITING_VALIDATION_STATUS = 'WAITING_VALIDATION' -export const REGISTERED_STATUS = 'REGISTERED' -const REINSTATED_STATUS = 'REINSTATED' -export const CERTIFIED_STATUS = 'CERTIFIED' -export const ISSUED_STATUS = 'ISSUED' -const REQUESTED_CORRECTION_STATUS = 'REQUESTED_CORRECTION' -const DECLARATION_UPDATED_STATUS = 'DECLARATION_UPDATED' - export const NOTIFICATION_TYPES = ['birth-notification', 'death-notification'] export const NAME_EN = 'en' -const validStatusMapping = { - [ARCHIVED_STATUS]: [ - DECLARED_STATUS, - REJECTED_STATUS, - VALIDATED_STATUS - ] as const, - [IN_PROGRESS_STATUS]: [null] as const, - [DECLARED_STATUS]: [ARCHIVED_STATUS, null] as const, - [REJECTED_STATUS]: [ - DECLARED_STATUS, - IN_PROGRESS_STATUS, - WAITING_VALIDATION_STATUS, - VALIDATED_STATUS, - ARCHIVED_STATUS - ] as const, - [VALIDATED_STATUS]: [ - DECLARED_STATUS, - IN_PROGRESS_STATUS, - REJECTED_STATUS, - ARCHIVED_STATUS, - DECLARATION_UPDATED_STATUS, - null - ] as const, - [WAITING_VALIDATION_STATUS]: [ - null, - DECLARED_STATUS, - IN_PROGRESS_STATUS, - REJECTED_STATUS, - VALIDATED_STATUS, - DECLARATION_UPDATED_STATUS - ] as const, - [REGISTERED_STATUS]: [ - null, - DECLARED_STATUS, - IN_PROGRESS_STATUS, - REJECTED_STATUS, - VALIDATED_STATUS, - WAITING_VALIDATION_STATUS - ] as const, - [CERTIFIED_STATUS]: [REGISTERED_STATUS, ISSUED_STATUS] as const, - [ISSUED_STATUS]: [CERTIFIED_STATUS] as const, - [REQUESTED_CORRECTION_STATUS]: [REGISTERED_STATUS, CERTIFIED_STATUS] as const, - [REINSTATED_STATUS]: [ARCHIVED_STATUS] as const -} - export interface ICorrection { section: string fieldName: string @@ -108,47 +47,6 @@ export interface IAssignment { officeName: string } -export interface IOperationHistory { - operationType: keyof typeof validStatusMapping - operatedOn: string -} - -export interface SearchDocument { - compositionId: string - compositionType?: string - event?: EVENT - type?: string - informantType?: string - contactNumber?: string - contactEmail?: string - dateOfDeclaration?: string - trackingId?: string - registrationNumber?: string - eventLocationId?: string - eventJurisdictionIds?: string[] - eventCountry?: string - declarationLocationId?: string - declarationJurisdictionIds?: string[] - rejectReason?: string - rejectComment?: string - relatesTo?: string[] - childFirstNames?: string - childFamilyName?: string - childFirstNamesLocal?: string - motherFirstNames?: string - motherFamilyName?: string - motherDoB?: string - motherIdentifier?: string - childDoB?: string - childIdentifier?: string - createdBy?: string - updatedBy?: string - createdAt?: string - modifiedAt?: string - assignment?: IAssignment | null - operationHistories?: IOperationHistory[] -} - export interface BirthDocument extends SearchDocument { childFirstNames?: string childMiddleName?: string @@ -293,9 +191,7 @@ export async function detectDeathDuplicates( export async function getCreatedBy(compositionId: string) { const results = await searchByCompositionId(compositionId, client) - const result = results?.body?.hits?.hits[0] - ?._source as unknown as SearchDocument - return result?.createdBy + return results?.body?.hits?.hits[0]?._source?.createdBy } export const composeOperationHistories = (bundle: SavedBundle) => { diff --git a/packages/search/src/features/registration/birth/service.ts b/packages/search/src/features/registration/birth/service.ts index b68333e1b5..961d048a97 100644 --- a/packages/search/src/features/registration/birth/service.ts +++ b/packages/search/src/features/registration/birth/service.ts @@ -13,11 +13,8 @@ import { searchByCompositionId } from '@search/elasticsearch/dbhelper' import { - EVENT, BirthDocument, NAME_EN, - IOperationHistory, - REJECTED_STATUS, composeOperationHistories, createStatusHistory, composeAssignment @@ -44,7 +41,10 @@ import { SavedBundle, getBusinessStatus, getInformantType, - ValidRecord + ValidRecord, + EVENT, + IOperationHistory, + REJECTED_STATUS } from '@opencrvs/commons/types' import { findAssignment } from '@opencrvs/commons/assignment' import { findPatientPrimaryIdentifier } from '@search/features/search/utils' @@ -75,8 +75,7 @@ export const composeDocument = ( createdAt: (existingDocument && existingDocument.body.hits.hits.length > 0 && - // @ts-ignore - existingDocument.body.hits.hits[0]?._source!.createdAt) || + existingDocument.body.hits.hits[0]?._source?.createdAt) || Date.now().toString(), modifiedAt: Date.now().toString(), operationHistories: composeOperationHistories(record) as IOperationHistory[] diff --git a/packages/search/src/features/registration/deduplicate/service.ts b/packages/search/src/features/registration/deduplicate/service.ts index b8b775be04..3c30e66d79 100644 --- a/packages/search/src/features/registration/deduplicate/service.ts +++ b/packages/search/src/features/registration/deduplicate/service.ts @@ -13,11 +13,7 @@ import { searchByCompositionId, updateComposition } from '@search/elasticsearch/dbhelper' -import { - BirthDocument, - SearchDocument, - DeathDocument -} from '@search/elasticsearch/utils' +import { BirthDocument, DeathDocument } from '@search/elasticsearch/utils' import { get } from 'lodash' import { ISearchResponse } from '@search/elasticsearch/client' import { OPENCRVS_INDEX_NAME } from '@search/constants' @@ -41,11 +37,15 @@ export const removeDuplicate = async ( if (!compositionId) { throw new Error('No Composition ID found') } + const composition = await searchByCompositionId(compositionId, client) - const body = get( - composition, - 'body.hits.hits[0]._source' - ) as unknown as SearchDocument + + const body = composition?.body.hits.hits[0]._source + + if (!body) { + throw new Error(`No composition found by ID ${compositionId}`) + } + body.relatesTo = extractRelatesToIDs(bundle) await updateComposition(compositionId, body, client) } diff --git a/packages/search/src/features/search/handler.ts b/packages/search/src/features/search/handler.ts index c894a0967b..e95837878d 100644 --- a/packages/search/src/features/search/handler.ts +++ b/packages/search/src/features/search/handler.ts @@ -9,14 +9,17 @@ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ import * as Hapi from '@hapi/hapi' -import { logger } from '@opencrvs/commons' +import { + logger, + SearchDocument, + EVENT, + getSearchTotalCount +} from '@opencrvs/commons' import { badRequest, internal } from '@hapi/boom' import { DEFAULT_SIZE, advancedSearch } from '@search/features/search/service' import { ISearchCriteria } from '@search/features/search/types' import { client } from '@search/elasticsearch/client' import { - SearchDocument, - EVENT, BirthDocument, DeathDocument, findDuplicateIds @@ -76,8 +79,8 @@ export async function getAllDocumentsHandler( ignore: [404] } ) - // @ts-ignore - const count: number = allDocumentsCountCheck?.body?.hits?.total?.value + + const count = getSearchTotalCount(allDocumentsCountCheck?.body?.hits?.total) if (count > 5000) { return internal( 'Elastic contains over 5000 results. It is risky to return all without pagination.' From 0925e07b4941d894a1d54840c60136c7b3fd4389 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 17 Jul 2024 12:20:55 +0300 Subject: [PATCH 11/15] chore(elastic-upgrade): cleanup deduplicate names and types --- .../search/src/features/delete/handler.ts | 3 +- .../registration/deduplicate/service.ts | 236 +++++++++--------- 2 files changed, 120 insertions(+), 119 deletions(-) diff --git a/packages/search/src/features/delete/handler.ts b/packages/search/src/features/delete/handler.ts index f9e216b74a..572d5f641e 100644 --- a/packages/search/src/features/delete/handler.ts +++ b/packages/search/src/features/delete/handler.ts @@ -18,9 +18,8 @@ export async function deleteOCRVSIndexHandler( request: Hapi.Request, h: Hapi.ResponseToolkit ) { - let response: any try { - response = await client.indices.delete( + const response = await client.indices.delete( { index: OPENCRVS_INDEX_NAME }, diff --git a/packages/search/src/features/registration/deduplicate/service.ts b/packages/search/src/features/registration/deduplicate/service.ts index 3c30e66d79..8b2e30da4c 100644 --- a/packages/search/src/features/registration/deduplicate/service.ts +++ b/packages/search/src/features/registration/deduplicate/service.ts @@ -28,6 +28,12 @@ import { } from 'date-fns' import * as elasticsearch from '@elastic/elasticsearch' +const isNonEmptyCondition = ( + value: T | null | undefined | '' +): value is T => { + return Boolean(value) +} + export const removeDuplicate = async ( bundle: fhir.Composition & { id: string }, client: elasticsearch.Client @@ -76,51 +82,49 @@ export const searchForBirthDuplicates = async ( return [] } - const motherMust: any = [ - // If mother identifier is provided, it needs to match 100% - body.motherIdentifier && { - match_phrase: { - motherIdentifier: body.motherIdentifier - } - }, - body.motherFirstNames && { - match: { - motherFirstNames: { - query: body.motherFirstNames, - fuzziness: FIRST_NAME_FUZZINESS - } - } - }, - body.motherFamilyName && { - match: { - motherFamilyName: { - query: body.motherFamilyName, - fuzziness: FIRST_NAME_FUZZINESS, - minimum_should_match: '100%' - } - } - }, - body.motherDoB && { - range: { - motherDoB: { - gte: subYears(new Date(body.motherDoB), 1).toISOString(), - lte: addYears(new Date(body.motherDoB), 1).toISOString() - } - } - }, - body.motherDoB && { - distance_feature: { - field: 'motherDoB', - pivot: '365d', - origin: new Date(body.motherDoB).toISOString(), - boost: 1.5 - } - } - ].filter(Boolean) - const mothersDetailsMatch = { bool: { - must: motherMust + must: [ + // If mother identifier is provided, it needs to match 100% + body.motherIdentifier && { + match_phrase: { + motherIdentifier: body.motherIdentifier + } + }, + body.motherFirstNames && { + match: { + motherFirstNames: { + query: body.motherFirstNames, + fuzziness: FIRST_NAME_FUZZINESS + } + } + }, + body.motherFamilyName && { + match: { + motherFamilyName: { + query: body.motherFamilyName, + fuzziness: FIRST_NAME_FUZZINESS, + minimum_should_match: '100%' + } + } + }, + body.motherDoB && { + range: { + motherDoB: { + gte: subYears(new Date(body.motherDoB), 1).toISOString(), + lte: addYears(new Date(body.motherDoB), 1).toISOString() + } + } + }, + body.motherDoB && { + distance_feature: { + field: 'motherDoB', + pivot: '365d', + origin: new Date(body.motherDoB).toISOString(), + boost: 1.5 + } + } + ].filter(isNonEmptyCondition) } } @@ -143,31 +147,39 @@ export const searchForBirthDuplicates = async ( boost: 1 } } - ].filter(Boolean) + ].filter(isNonEmptyCondition) } } - const must1: any = [ - body.childFirstNames && { - match: { - childFirstNames: { - query: body.childFirstNames, - fuzziness: FIRST_NAME_FUZZINESS - } - } - }, - body.childFamilyName && { - match: { - childFamilyName: { - query: body.childFamilyName, - fuzziness: FIRST_NAME_FUZZINESS, - minimum_should_match: '100%' + const childsNameMatch = { + bool: { + must: { + bool: { + must: [ + body.childFirstNames && { + match: { + childFirstNames: { + query: body.childFirstNames, + fuzziness: FIRST_NAME_FUZZINESS + } + } + }, + body.childFamilyName && { + match: { + childFamilyName: { + query: body.childFamilyName, + fuzziness: FIRST_NAME_FUZZINESS, + minimum_should_match: '100%' + } + } + } + ].filter(isNonEmptyCondition) } } } - ].filter(Boolean) + } - const bool1 = body.childDoB && { + const childsBirthWithinRange = body.childDoB && { bool: { should: [ { @@ -196,7 +208,6 @@ export const searchForBirthDuplicates = async ( } } - const must2: any = [bool1, mothersDetailsMatch].filter(Boolean) try { const result = await client.search( { @@ -212,13 +223,10 @@ export const searchForBirthDuplicates = async ( { bool: { must: [ - { - bool: { - must: must1 - } - }, - ...must2 - ].filter(Boolean) + childsNameMatch, + childsBirthWithinRange, + mothersDetailsMatch + ].filter(isNonEmptyCondition) } } ] @@ -252,7 +260,7 @@ export const searchForDeathDuplicates = async ( return [] } - const must1: any = [ + const deceasedsDetailsMatch = [ body.deceasedFirstNames && { match: { deceasedFirstNames: { @@ -274,28 +282,45 @@ export const searchForDeathDuplicates = async ( deceasedIdentifier: body.deceasedIdentifier } } - ].filter((clause) => !!clause) + ].filter(isNonEmptyCondition) - const must2 = body.deathDate - ? [ - { - range: { - deathDate: { - gte: subDays(new Date(body.deathDate), 5).toISOString(), - lte: addDays(new Date(body.deathDate), 5).toISOString() - } - } - }, - { - distance_feature: { - field: 'deathDate', - pivot: '5d', // 5 days - origin: new Date(body.deathDate).toISOString(), - boost: 1 - } + const deathDateWithinRange = [ + body.deathDate && { + range: { + deathDate: { + gte: subDays(new Date(body.deathDate), 5).toISOString(), + lte: addDays(new Date(body.deathDate), 5).toISOString() } - ] - : [] + } + }, + body.deathDate && { + distance_feature: { + field: 'deathDate', + pivot: '5d', // 5 days + origin: new Date(body.deathDate).toISOString(), + boost: 1 + } + } + ].filter(isNonEmptyCondition) + + const birthDateWithinRange = [ + body.deceasedDoB && { + range: { + deceasedDoB: { + gte: subDays(new Date(body.deceasedDoB), 5).toISOString(), + lte: addDays(new Date(body.deceasedDoB), 5).toISOString() + } + } + }, + body.deceasedDoB && { + distance_feature: { + field: 'deceasedDoB', + pivot: '5d', // 5 days + origin: new Date(body.deceasedDoB).toISOString(), + boost: 1 + } + } + ].filter(isNonEmptyCondition) try { const result = await client.search( @@ -304,41 +329,18 @@ export const searchForDeathDuplicates = async ( query: { bool: { must: [ - ...must1, + ...deceasedsDetailsMatch, { bool: { - must: must2 + must: deathDateWithinRange } }, { bool: { - must: [ - body.deceasedDoB && { - range: { - deceasedDoB: { - gte: subDays( - new Date(body.deceasedDoB), - 5 - ).toISOString(), - lte: addDays( - new Date(body.deceasedDoB), - 5 - ).toISOString() - } - } - }, - body.deceasedDoB && { - distance_feature: { - field: 'deceasedDoB', - pivot: '5d', // 5 days - origin: new Date(body.deceasedDoB).toISOString(), - boost: 1 - } - } - ].filter(Boolean) + must: birthDateWithinRange } } - ].filter(Boolean) + ] } } }, From 49b0707ce11f1ce7342e3328f79e87d0fc5f64ee Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 17 Jul 2024 12:27:44 +0300 Subject: [PATCH 12/15] chore(elastic-upgrade): clean up returns and any type --- packages/migration/src/utils/elasticsearch-helper.ts | 3 +-- packages/search/src/features/search/handler.ts | 2 +- packages/search/src/features/search/service.ts | 8 +++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index bd8d5f7ff1..73f0cee6f3 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -42,9 +42,8 @@ export const updateComposition = async ( ) } catch (e) { console.error(`updateComposition: error: ${e}`) + return } - - return } export const renameField = async ( diff --git a/packages/search/src/features/search/handler.ts b/packages/search/src/features/search/handler.ts index e95837878d..6910de7d82 100644 --- a/packages/search/src/features/search/handler.ts +++ b/packages/search/src/features/search/handler.ts @@ -140,7 +140,7 @@ export async function getStatusWiseRegistrationCountHandler( } const response = await client.search< - any, + SearchDocument, { statusCounts: { buckets: Array<{ diff --git a/packages/search/src/features/search/service.ts b/packages/search/src/features/search/service.ts index 39ae68bcb2..10602a9b65 100644 --- a/packages/search/src/features/search/service.ts +++ b/packages/search/src/features/search/service.ts @@ -59,9 +59,9 @@ export const advancedSearch = async ( payload: ISearchCriteria ) => { const formattedParams = await formatSearchParams(payload, isExternalSearch) - let response + try { - response = await client.search(formattedParams, { + return await client.search(formattedParams, { ignore: !isExternalSearch ? [404] : undefined, meta: true }) @@ -71,8 +71,6 @@ export const advancedSearch = async ( } else { logger.error('Search error: ', error) } - return undefined + return } - - return response } From f9cdd6ff6748c43e347d54edd8f366d8dd7a7da2 Mon Sep 17 00:00:00 2001 From: Markus Date: Wed, 17 Jul 2024 12:48:14 +0300 Subject: [PATCH 13/15] chore(elastic-upgrade): import from right file, simplify conditionals --- packages/search/src/elasticsearch/dbhelper.ts | 4 ++-- packages/search/src/features/fhir/fhir-utils.ts | 4 ++-- .../features/registration/assignment/service.ts | 4 ++-- .../src/features/registration/birth/service.ts | 4 +--- .../src/features/registration/death/service.ts | 12 +++++------- .../src/features/registration/marriage/service.ts | 14 ++++++-------- packages/search/src/features/reindex/reindex.ts | 8 ++++++-- packages/search/src/features/search/utils.ts | 6 ++---- 8 files changed, 26 insertions(+), 30 deletions(-) diff --git a/packages/search/src/elasticsearch/dbhelper.ts b/packages/search/src/elasticsearch/dbhelper.ts index f5af6d5c57..5ae14f55dd 100644 --- a/packages/search/src/elasticsearch/dbhelper.ts +++ b/packages/search/src/elasticsearch/dbhelper.ts @@ -57,9 +57,9 @@ export const updateComposition = async ( ) } catch (e) { logger.error(`updateComposition: error: ${e}`) - } - return + return + } } export const searchByCompositionId = async ( diff --git a/packages/search/src/features/fhir/fhir-utils.ts b/packages/search/src/features/fhir/fhir-utils.ts index 781a0cdf76..3f6462e5ed 100644 --- a/packages/search/src/features/fhir/fhir-utils.ts +++ b/packages/search/src/features/fhir/fhir-utils.ts @@ -25,10 +25,10 @@ import { SavedComposition, SavedLocation, SavedTask, - TaskHistory + TaskHistory, + SearchDocument } from '@opencrvs/commons/types' import { FHIR_URL } from '@search/constants' -import { SearchDocument } from '@search/elasticsearch/utils' import { logger } from '@opencrvs/commons' import fetch from 'node-fetch' diff --git a/packages/search/src/features/registration/assignment/service.ts b/packages/search/src/features/registration/assignment/service.ts index 53f51caef2..138547d40d 100644 --- a/packages/search/src/features/registration/assignment/service.ts +++ b/packages/search/src/features/registration/assignment/service.ts @@ -12,7 +12,6 @@ import { updateComposition } from '@search/elasticsearch/dbhelper' import { getUser, IAssignment, - SearchDocument, IUserModelData, NAME_EN } from '@search/elasticsearch/utils' @@ -25,7 +24,8 @@ import { getTaskFromSavedBundle, LAST_USER_EXTENSION_URL, resourceIdentifierToUUID, - SavedBundle + SavedBundle, + SearchDocument } from '@opencrvs/commons/types' export async function updateEventToAddAssignment(requestBundle: Hapi.Request) { diff --git a/packages/search/src/features/registration/birth/service.ts b/packages/search/src/features/registration/birth/service.ts index 961d048a97..5c6a03673c 100644 --- a/packages/search/src/features/registration/birth/service.ts +++ b/packages/search/src/features/registration/birth/service.ts @@ -73,9 +73,7 @@ export const composeDocument = ( compositionId: composition.id, event: EVENT.BIRTH, createdAt: - (existingDocument && - existingDocument.body.hits.hits.length > 0 && - existingDocument.body.hits.hits[0]?._source?.createdAt) || + existingDocument?.body?.hits?.hits?.[0]?._source?.createdAt || Date.now().toString(), modifiedAt: Date.now().toString(), operationHistories: composeOperationHistories(record) as IOperationHistory[] diff --git a/packages/search/src/features/registration/death/service.ts b/packages/search/src/features/registration/death/service.ts index 37d39af22f..a4953b071e 100644 --- a/packages/search/src/features/registration/death/service.ts +++ b/packages/search/src/features/registration/death/service.ts @@ -15,11 +15,8 @@ import { import { composeOperationHistories, createStatusHistory, - EVENT, DeathDocument, - IOperationHistory, NAME_EN, - REJECTED_STATUS, composeAssignment } from '@search/elasticsearch/utils' import { @@ -43,7 +40,10 @@ import { resourceIdentifierToUUID, findFirstTaskHistory, getInformantType, - ValidRecord + ValidRecord, + REJECTED_STATUS, + IOperationHistory, + EVENT } from '@opencrvs/commons/types' import { findAssignment } from '@opencrvs/commons/assignment' import { findPatientPrimaryIdentifier } from '@search/features/search/utils' @@ -66,9 +66,7 @@ export const composeDocument = ( compositionId: composition.id, event: EVENT.DEATH, createdAt: - (existingDocument && - existingDocument.body.hits.hits.length > 0 && - existingDocument.body.hits.hits[0]?._source?.createdAt) || + existingDocument?.body?.hits?.hits?.[0]?._source?.createdAt || Date.now().toString(), operationHistories: composeOperationHistories(bundle) as IOperationHistory[] } diff --git a/packages/search/src/features/registration/marriage/service.ts b/packages/search/src/features/registration/marriage/service.ts index 61d7265218..727a527fd5 100644 --- a/packages/search/src/features/registration/marriage/service.ts +++ b/packages/search/src/features/registration/marriage/service.ts @@ -14,12 +14,8 @@ import { } from '@search/elasticsearch/dbhelper' import { createStatusHistory, - EVENT, - SearchDocument, MarriageDocument, - IOperationHistory, NAME_EN, - REJECTED_STATUS, composeOperationHistories, composeAssignment } from '@search/elasticsearch/utils' @@ -45,7 +41,11 @@ import { SavedRelatedPerson, findFirstTaskHistory, getInformantType, - ValidRecord + ValidRecord, + EVENT, + SearchDocument, + IOperationHistory, + REJECTED_STATUS } from '@opencrvs/commons/types' import { findAssignment } from '@opencrvs/commons/assignment' import { findPatientPrimaryIdentifier } from '@search/features/search/utils' @@ -66,9 +66,7 @@ export const composeDocument = ( compositionId: composition.id, event: EVENT.MARRIAGE, createdAt: - (existingDocument && - existingDocument.body.hits.hits.length > 0 && - existingDocument.body.hits.hits[0]?._source?.createdAt) || + existingDocument?.body?.hits?.hits?.[0]?._source?.createdAt || Date.now().toString(), operationHistories: composeOperationHistories(bundle) as IOperationHistory[] } diff --git a/packages/search/src/features/reindex/reindex.ts b/packages/search/src/features/reindex/reindex.ts index 3ad6141242..3295e0c84e 100644 --- a/packages/search/src/features/reindex/reindex.ts +++ b/packages/search/src/features/reindex/reindex.ts @@ -8,10 +8,14 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { EVENT_TYPE, ValidRecord } from '@opencrvs/commons/types' +import { + EVENT_TYPE, + ValidRecord, + SearchDocument +} from '@opencrvs/commons/types' import { OPENCRVS_INDEX_NAME } from '@search/constants' import { client } from '@search/elasticsearch/client' -import { BirthDocument, SearchDocument } from '@search/elasticsearch/utils' +import { BirthDocument } from '@search/elasticsearch/utils' import { streamAllRecords } from '@search/features/records/service' import { composeDocument as composeBirthDocument } from '@search/features/registration/birth/service' import { composeDocument as composeDeathDocument } from '@search/features/registration/death/service' diff --git a/packages/search/src/features/search/utils.ts b/packages/search/src/features/search/utils.ts index 59ebdee166..2149b71d95 100644 --- a/packages/search/src/features/search/utils.ts +++ b/packages/search/src/features/search/utils.ts @@ -11,12 +11,10 @@ import { findPatientIdentifier, Patient, - SUPPORTED_PATIENT_IDENTIFIER_CODES -} from '@opencrvs/commons/types' -import { + SUPPORTED_PATIENT_IDENTIFIER_CODES, CERTIFIED_STATUS, REGISTERED_STATUS -} from '@search/elasticsearch/utils' +} from '@opencrvs/commons/types' import { IAdvancedSearchParam } from '@search/features/search/types' import { transformDeprecatedParamsToSupported } from './deprecation-support' import { resolveLocationChildren } from './location' From 82a672b1d9eb269d9d3c37b7f2b5c4d874eb2d90 Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 25 Jul 2024 09:33:17 +0300 Subject: [PATCH 14/15] chore(elastic-upgrade): remove accidental nesting of must bools --- .../registration/deduplicate/service.ts | 38 +++++++++---------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/packages/search/src/features/registration/deduplicate/service.ts b/packages/search/src/features/registration/deduplicate/service.ts index 8b2e30da4c..7c1b020074 100644 --- a/packages/search/src/features/registration/deduplicate/service.ts +++ b/packages/search/src/features/registration/deduplicate/service.ts @@ -153,29 +153,25 @@ export const searchForBirthDuplicates = async ( const childsNameMatch = { bool: { - must: { - bool: { - must: [ - body.childFirstNames && { - match: { - childFirstNames: { - query: body.childFirstNames, - fuzziness: FIRST_NAME_FUZZINESS - } - } - }, - body.childFamilyName && { - match: { - childFamilyName: { - query: body.childFamilyName, - fuzziness: FIRST_NAME_FUZZINESS, - minimum_should_match: '100%' - } - } + must: [ + body.childFirstNames && { + match: { + childFirstNames: { + query: body.childFirstNames, + fuzziness: FIRST_NAME_FUZZINESS + } + } + }, + body.childFamilyName && { + match: { + childFamilyName: { + query: body.childFamilyName, + fuzziness: FIRST_NAME_FUZZINESS, + minimum_should_match: '100%' } - ].filter(isNonEmptyCondition) + } } - } + ].filter(isNonEmptyCondition) } } From 6645e83784fe07ca26d67d7f5a6b24d4edc6b6c3 Mon Sep 17 00:00:00 2001 From: Markus Date: Thu, 25 Jul 2024 09:37:27 +0300 Subject: [PATCH 15/15] chore(elastic-upgrade): remove commented out code compositions were removed already from db-helper earlier --- packages/migration/src/utils/elasticsearch-helper.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index 73f0cee6f3..cd74b45828 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -29,7 +29,6 @@ export const updateComposition = async ( return await client.update( { index: ELASTICSEARCH_INDEX_NAME, - // type: 'compositions', @todo: check whether this should work id, body: { doc: body @@ -107,7 +106,6 @@ export const searchCompositionByCriteria = async ( return await client.search( { index: ELASTICSEARCH_INDEX_NAME, - // type: 'compositions', @todo: check whether this should work body: { query: criteriaObject, ...extraConfigs