Improve full query condition

src/common/scientific-relation.enum.ts

@@ -3,4 +3,5 @@ export enum ScientificRelation {
   EQUAL_TO_NUMERIC = "EQUAL_TO_NUMERIC",
   GREATER_THAN = "GREATER_THAN",
   LESS_THAN = "LESS_THAN",
+  CONTAINS_STRING = "CONTAINS_STRING",
 }

src/common/utils.ts

@@ -86,11 +86,26 @@ export const convertToRequestedUnit = (
   };
 };
 
+const buildCondition = (
+  key: string,
+  value: string | number,
+  operator: string,
+): Record<string, unknown> => {
+  const conditions: Record<string, unknown> = { $or: [] };
+  conditions["$or"] = ["", ".v", ".value"].map((suffix) => {
+    return {
+      [`${key}${suffix}`]: { [`${operator}`]: value },
+    };
+  });
+  return conditions;
+};
+
 export const mapScientificQuery = (
   key: string,
   scientific: IScientificFilter[] = [],
 ): Record<string, unknown> => {
   const scientificFilterQuery: Record<string, unknown> = {};
+  const scientificFilterQueryOr: Record<string, unknown>[] = [];
 
   const keyToFieldMapping: Record<string, string> = {
     scientific: "scientificMetadata",
@@ -112,7 +127,9 @@ export const mapScientificQuery = (
 
     switch (relation) {
       case ScientificRelation.EQUAL_TO_STRING: {
-        scientificFilterQuery[`${matchKeyGeneric}.value`] = { $eq: rhs };
+        scientificFilterQueryOr.push(
+          buildCondition(matchKeyGeneric, rhs, "$eq"),
+        );
         break;
       }
       case ScientificRelation.EQUAL_TO_NUMERIC: {
@@ -131,7 +148,9 @@ export const mapScientificQuery = (
           scientificFilterQuery[matchKeyMeasurement] = { $gt: valueSI };
           scientificFilterQuery[matchUnit] = { $eq: unitSI };
         } else {
-          scientificFilterQuery[`${matchKeyGeneric}.value`] = { $gt: rhs };
+          scientificFilterQueryOr.push(
+            buildCondition(matchKeyGeneric, rhs, "$gt"),
+          );
         }
         break;
       }
@@ -141,12 +160,26 @@ export const mapScientificQuery = (
           scientificFilterQuery[matchKeyMeasurement] = { $lt: valueSI };
           scientificFilterQuery[matchUnit] = { $eq: unitSI };
         } else {
-          scientificFilterQuery[`${matchKeyGeneric}.value`] = { $lt: rhs };
+          scientificFilterQueryOr.push(
+            buildCondition(matchKeyGeneric, rhs, "$lt"),
+          );
         }
         break;
       }
+      case ScientificRelation.CONTAINS_STRING: {
+        scientificFilterQueryOr.push(
+          buildCondition(matchKeyGeneric, rhs, `/${rhs}/`),
+        );
+        break;
+      }
     }
   });
+  if (scientificFilterQueryOr.length == 1) {
+    scientificFilterQuery["$or"] = scientificFilterQueryOr[0]["$or"];
+  } else if (scientificFilterQueryOr.length > 1) {
+    scientificFilterQuery["$and"] = scientificFilterQueryOr;
+  }
+
   return scientificFilterQuery;
 };
 

src/elastic-search/configuration/datasetFieldMapping.ts

@@ -30,18 +30,13 @@ export const datasetMappings: MappingObject = {
   creationTime: {
     type: "date",
   },
+  endTime: {
+    type: "date",
+  },
   scientificMetadata: {
     type: "nested",
     dynamic: true,
-    properties: {
-      runNumber: {
-        properties: {
-          value: {
-            type: "long",
-          },
-        },
-      },
-    },
+    properties: {},
   },
   history: {
     type: "nested",

src/elastic-search/configuration/indexSetting.ts

@@ -22,13 +22,25 @@ export const special_character_filter: AnalysisPatternReplaceCharFilter = {
 
 //Dynamic templates
 export const dynamic_template: Record<string, MappingDynamicTemplate>[] = [
+  // NOTE: date as keyword is temporary solution for date format inconsistency issue in the scientificMetadata field
   {
-    string_as_keyword: {
+    date_as_keyword: {
       path_match: "scientificMetadata.*.*",
-      match_mapping_type: "string",
+      match_mapping_type: "date",
+      mapping: {
+        type: "keyword",
+      },
+    },
+  },
+  // NOTE: This is a workaround for the issue where the start_time field is not being
+  // parsed correctly. This is a temporary solution until
+  // we can find a better way to handle date format.
+  {
+    start_time_as_keyword: {
+      path_match: "scientificMetadata.start_time.*",
+      match_mapping_type: "long",
       mapping: {
         type: "keyword",
-        ignore_above: 256,
       },
     },
   },
@@ -44,9 +56,20 @@ export const dynamic_template: Record<string, MappingDynamicTemplate>[] = [
     },
   },
   {
-    date_as_keyword: {
+    double_as_double: {
       path_match: "scientificMetadata.*.*",
-      match_mapping_type: "date",
+      match_mapping_type: "double",
+      mapping: {
+        type: "double",
+        coerce: true,
+        ignore_malformed: true,
+      },
+    },
+  },
+  {
+    string_as_keyword: {
+      path_match: "scientificMetadata.*.*",
+      match_mapping_type: "string",
       mapping: {
         type: "keyword",
         ignore_above: 256,

src/elastic-search/elastic-search.service.ts

@@ -26,9 +26,9 @@ import {
 import { ConfigService } from "@nestjs/config";
 import { sleep } from "src/common/utils";
 import {
-  transformKeysInObject,
   initialSyncTransform,
   transformFacets,
+  addValueType,
 } from "./helpers/utils";
 
 import { SortFields } from "./providers/fields.enum";
@@ -86,6 +86,10 @@ export class ElasticSearchService implements OnModuleInit {
       const isIndexExists = await this.isIndexExists(this.defaultIndex);
       if (!isIndexExists) {
         await this.createIndex(this.defaultIndex);
+        Logger.log(
+          `New index ${this.defaultIndex}is created `,
+          "ElasticSearch",
+        );
       }
       this.connected = true;
       Logger.log("Elasticsearch Connected", "ElasticSearch");
@@ -141,26 +145,18 @@ export class ElasticSearchService implements OnModuleInit {
         index,
         body: {
           settings: defaultElasticSettings,
+          mappings: {
+            dynamic: true,
+            dynamic_templates: dynamic_template,
+            numeric_detection: true,
+            date_detection: true,
+            dynamic_date_formats: [
+              "yyyy-MM-dd'T'HH:mm:ss|| yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZ||yyyy-MM-dd'T'HH:mm:ss.SSS'Z'||yyyy-MM-dd'T'HH:mm:ss.SSS",
+            ],
+            properties: datasetMappings,
+          },
         },
       });
-      await this.esService.indices.close({ index });
-      await this.esService.indices.putSettings({
-        index,
-        body: {
-          settings: defaultElasticSettings,
-        },
-      });
-      await this.esService.indices.putMapping({
-        index,
-        dynamic: true,
-        body: {
-          dynamic_templates: dynamic_template,
-          properties: datasetMappings,
-        },
-      });
-      await this.esService.indices.open({
-        index,
-      });
       Logger.log(
         `Elasticsearch Index Created-> Index: ${index}`,
         "Elasticsearch",
@@ -362,7 +358,7 @@ export class ElasticSearchService implements OnModuleInit {
   async updateInsertDocument(data: Partial<DatasetDocument>) {
     //NOTE: Replace all keys with lower case, also replace spaces and dot with underscore
     delete data._id;
-    const transformedScientificMetadata = transformKeysInObject(
+    const transformedScientificMetadata = addValueType(
       data.scientificMetadata as Record<string, unknown>,
     );