diff --git a/.github/ISSUE_TEMPLATE/issue-.md b/.github/ISSUE_TEMPLATE/issue-.md deleted file mode 100644 index 60f04d42d..000000000 --- a/.github/ISSUE_TEMPLATE/issue-.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: 'Issue ' -about: Describe this issue purpose here. - ---- - -## Issue Name - -### Summary - -### Steps to Reproduce - -### Current Behaviour - -### Expected Behaviour - -### Extra Details - -Here you should include details about the system (if it is unique) and possible information about a fix (feel free to link to code where relevant). Screenshots/GIFs are also fine here. diff --git a/src/common/scientific-relation.enum.ts b/src/common/scientific-relation.enum.ts index 1eb550bf0..e9acf37b9 100644 --- a/src/common/scientific-relation.enum.ts +++ b/src/common/scientific-relation.enum.ts @@ -3,4 +3,5 @@ export enum ScientificRelation { EQUAL_TO_NUMERIC = "EQUAL_TO_NUMERIC", GREATER_THAN = "GREATER_THAN", LESS_THAN = "LESS_THAN", + CONTAINS_STRING = "CONTAINS_STRING", } diff --git a/src/common/utils.ts b/src/common/utils.ts index 7e16add43..3a872ff3f 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -86,11 +86,26 @@ export const convertToRequestedUnit = ( }; }; +const buildCondition = ( + key: string, + value: string | number, + operator: string, +): Record => { + const conditions: Record = { $or: [] }; + conditions["$or"] = ["", ".v", ".value"].map((suffix) => { + return { + [`${key}${suffix}`]: { [`${operator}`]: value }, + }; + }); + return conditions; +}; + export const mapScientificQuery = ( key: string, scientific: IScientificFilter[] = [], ): Record => { const scientificFilterQuery: Record = {}; + const scientificFilterQueryOr: Record[] = []; const keyToFieldMapping: Record = { scientific: "scientificMetadata", @@ -112,7 +127,9 @@ export const mapScientificQuery = ( switch (relation) { case ScientificRelation.EQUAL_TO_STRING: { - scientificFilterQuery[`${matchKeyGeneric}.value`] = { $eq: rhs }; + scientificFilterQueryOr.push( + buildCondition(matchKeyGeneric, rhs, "$eq"), + ); break; } case ScientificRelation.EQUAL_TO_NUMERIC: { @@ -131,7 +148,9 @@ export const mapScientificQuery = ( scientificFilterQuery[matchKeyMeasurement] = { $gt: valueSI }; scientificFilterQuery[matchUnit] = { $eq: unitSI }; } else { - scientificFilterQuery[`${matchKeyGeneric}.value`] = { $gt: rhs }; + scientificFilterQueryOr.push( + buildCondition(matchKeyGeneric, rhs, "$gt"), + ); } break; } @@ -141,12 +160,26 @@ export const mapScientificQuery = ( scientificFilterQuery[matchKeyMeasurement] = { $lt: valueSI }; scientificFilterQuery[matchUnit] = { $eq: unitSI }; } else { - scientificFilterQuery[`${matchKeyGeneric}.value`] = { $lt: rhs }; + scientificFilterQueryOr.push( + buildCondition(matchKeyGeneric, rhs, "$lt"), + ); } break; } + case ScientificRelation.CONTAINS_STRING: { + scientificFilterQueryOr.push( + buildCondition(matchKeyGeneric, rhs, `/${rhs}/`), + ); + break; + } } }); + if (scientificFilterQueryOr.length == 1) { + scientificFilterQuery["$or"] = scientificFilterQueryOr[0]["$or"]; + } else if (scientificFilterQueryOr.length > 1) { + scientificFilterQuery["$and"] = scientificFilterQueryOr; + } + return scientificFilterQuery; }; diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts index 988bb5b7e..429555ee8 100644 --- a/src/elastic-search/configuration/datasetFieldMapping.ts +++ b/src/elastic-search/configuration/datasetFieldMapping.ts @@ -30,18 +30,13 @@ export const datasetMappings: MappingObject = { creationTime: { type: "date", }, + endTime: { + type: "date", + }, scientificMetadata: { type: "nested", dynamic: true, - properties: { - runNumber: { - properties: { - value: { - type: "long", - }, - }, - }, - }, + properties: {}, }, history: { type: "nested", diff --git a/src/elastic-search/configuration/indexSetting.ts b/src/elastic-search/configuration/indexSetting.ts index bbb60a9f2..7fc473aae 100644 --- a/src/elastic-search/configuration/indexSetting.ts +++ b/src/elastic-search/configuration/indexSetting.ts @@ -22,13 +22,25 @@ export const special_character_filter: AnalysisPatternReplaceCharFilter = { //Dynamic templates export const dynamic_template: Record[] = [ + // NOTE: date as keyword is temporary solution for date format inconsistency issue in the scientificMetadata field { - string_as_keyword: { + date_as_keyword: { path_match: "scientificMetadata.*.*", - match_mapping_type: "string", + match_mapping_type: "date", + mapping: { + type: "keyword", + }, + }, + }, + // NOTE: This is a workaround for the issue where the start_time field is not being + // parsed correctly. This is a temporary solution until + // we can find a better way to handle date format. + { + start_time_as_keyword: { + path_match: "scientificMetadata.start_time.*", + match_mapping_type: "long", mapping: { type: "keyword", - ignore_above: 256, }, }, }, @@ -44,9 +56,20 @@ export const dynamic_template: Record[] = [ }, }, { - date_as_keyword: { + double_as_double: { path_match: "scientificMetadata.*.*", - match_mapping_type: "date", + match_mapping_type: "double", + mapping: { + type: "double", + coerce: true, + ignore_malformed: true, + }, + }, + }, + { + string_as_keyword: { + path_match: "scientificMetadata.*.*", + match_mapping_type: "string", mapping: { type: "keyword", ignore_above: 256, diff --git a/src/elastic-search/elastic-search.service.ts b/src/elastic-search/elastic-search.service.ts index 69782558b..12a02353f 100644 --- a/src/elastic-search/elastic-search.service.ts +++ b/src/elastic-search/elastic-search.service.ts @@ -26,9 +26,9 @@ import { import { ConfigService } from "@nestjs/config"; import { sleep } from "src/common/utils"; import { - transformKeysInObject, initialSyncTransform, transformFacets, + addValueType, } from "./helpers/utils"; import { SortFields } from "./providers/fields.enum"; @@ -86,6 +86,10 @@ export class ElasticSearchService implements OnModuleInit { const isIndexExists = await this.isIndexExists(this.defaultIndex); if (!isIndexExists) { await this.createIndex(this.defaultIndex); + Logger.log( + `New index ${this.defaultIndex}is created `, + "ElasticSearch", + ); } this.connected = true; Logger.log("Elasticsearch Connected", "ElasticSearch"); @@ -141,26 +145,18 @@ export class ElasticSearchService implements OnModuleInit { index, body: { settings: defaultElasticSettings, + mappings: { + dynamic: true, + dynamic_templates: dynamic_template, + numeric_detection: true, + date_detection: true, + dynamic_date_formats: [ + "yyyy-MM-dd'T'HH:mm:ss|| yyyy-MM-dd HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZ||yyyy-MM-dd'T'HH:mm:ss.SSS'Z'||yyyy-MM-dd'T'HH:mm:ss.SSS", + ], + properties: datasetMappings, + }, }, }); - await this.esService.indices.close({ index }); - await this.esService.indices.putSettings({ - index, - body: { - settings: defaultElasticSettings, - }, - }); - await this.esService.indices.putMapping({ - index, - dynamic: true, - body: { - dynamic_templates: dynamic_template, - properties: datasetMappings, - }, - }); - await this.esService.indices.open({ - index, - }); Logger.log( `Elasticsearch Index Created-> Index: ${index}`, "Elasticsearch", @@ -362,7 +358,7 @@ export class ElasticSearchService implements OnModuleInit { async updateInsertDocument(data: Partial) { //NOTE: Replace all keys with lower case, also replace spaces and dot with underscore delete data._id; - const transformedScientificMetadata = transformKeysInObject( + const transformedScientificMetadata = addValueType( data.scientificMetadata as Record, ); diff --git a/src/elastic-search/helpers/utils.ts b/src/elastic-search/helpers/utils.ts index 0a0198b4b..9c8d28242 100644 --- a/src/elastic-search/helpers/utils.ts +++ b/src/elastic-search/helpers/utils.ts @@ -3,12 +3,17 @@ import { AggregationsFrequentItemSetsBucketKeys, } from "@elastic/elasticsearch/lib/api/types"; import { DatasetClass } from "src/datasets/schemas/dataset.schema"; -import { IFilter, ITransformedFullFacets } from "../interfaces/es-common.type"; +import { + IFilter, + ITransformedFullFacets, + nestedQueryObject, + ScientificQuery, +} from "../interfaces/es-common.type"; export const transformKey = (key: string): string => { return key.trim().replace(/[.]/g, "\\.").replace(/ /g, "_").toLowerCase(); }; -export const transformKeysInObject = (obj: Record) => { +export const addValueType = (obj: Record) => { const newObj: Record = {}; for (const [key, value] of Object.entries(obj)) { @@ -16,6 +21,7 @@ export const transformKeysInObject = (obj: Record) => { const isNumberValueType = typeof (value as Record)?.value === "number"; + if (isNumberValueType) { (value as Record)["value_type"] = "number"; } else { @@ -66,51 +72,103 @@ export const initialSyncTransform = (obj: DatasetClass) => { return modifiedDocInObject; }; +const extractNestedQueryOperationValue = (query: nestedQueryObject) => { + const field = Object.keys(query)[0]; + const operationWithPrefix = Object.keys(query[field])[0]; + + const value = + typeof query[field][operationWithPrefix] === "string" + ? (query[field][operationWithPrefix] as string).trim() + : query[field][operationWithPrefix]; + + const operation = operationWithPrefix.replace("$", ""); + + return { operation, value, field }; +}; + export const convertToElasticSearchQuery = ( - scientificQuery: Record, -) => { + scientificQuery: ScientificQuery, +): IFilter[] => { const filters: IFilter[] = []; for (const field in scientificQuery) { - const query = scientificQuery[field] as Record; - const operation = Object.keys(query)[0]; - const value = - typeof query[operation] === "string" - ? (query[operation] as string).trim() - : query[operation]; - - const esOperation = operation.replace("$", ""); - - // NOTE-EXAMPLE: - // trasnformedKey = "scientificMetadata.someKey.value" - // firstPart = "scientificMetadata", - // middlePart = "someKey" - const { transformedKey, firstPart, middlePart } = transformMiddleKey(field); - - let filter = {}; - - const fieldType = field.split(".").pop(); - - if (fieldType === "valueSI" || fieldType === "value") { - const numberFilter = { - term: { - [`${firstPart}.${middlePart}.value_type`]: - typeof value === "number" ? "number" : "string", + const query = scientificQuery[field]; + + if (field === "$and" && Array.isArray(query)) { + query.forEach((query: { $or: nestedQueryObject[] }) => { + const shouldQueries = query.$or.map((orQuery: nestedQueryObject) => { + const { operation, value, field } = + extractNestedQueryOperationValue(orQuery); + const filterType = operation === "eq" ? "term" : "range"; + return { + [filterType]: { + [field]: operation === "eq" ? value : { [operation]: value }, + }, + }; + }); + filters.push({ + bool: { + should: shouldQueries, + minimum_should_match: 1, + }, + }); + }); + } else if (field === "$or" && Array.isArray(query)) { + const shouldQueries = query.map((query: nestedQueryObject) => { + const { operation, value, field } = + extractNestedQueryOperationValue(query); + const filterType = operation === "eq" ? "term" : "range"; + return { + [filterType]: { + [field]: operation === "eq" ? value : { [operation]: value }, + }, + }; + }); + filters.push({ + bool: { + should: shouldQueries, + minimum_should_match: 1, }, - }; - filters.push(numberFilter); + }); + } else { + const operation = Object.keys(query)[0]; + + const value = + typeof (query as Record)[operation] === "string" + ? (query as Record)[operation].trim() + : (query as Record)[operation]; + const esOperation = operation.replace("$", ""); + + // NOTE: + // trasnformedKey = "scientificMetadata.someKey.value" + // firstPart = "scientificMetadata", + // middlePart = "someKey" + // lastPart = "value" + const { transformedKey, firstPart, middlePart, lastPart } = + transformMiddleKey(field); + + if (lastPart === "valueSI" || lastPart === "value") { + const numberFilter = { + term: { + [`${firstPart}.${middlePart}.value_type`]: + typeof value === "number" ? "number" : "string", + }, + }; + filters.push(numberFilter); + } + + const filter = + esOperation === "eq" + ? { + term: { [`${transformedKey}`]: value }, + } + : { + range: { [`${transformedKey}`]: { [esOperation]: value } }, + }; + + filters.push(filter); } - - filter = - esOperation === "eq" - ? { - term: { [`${transformedKey}`]: value }, - } - : { range: { [`${transformedKey}`]: { [esOperation]: value } } }; - - filters.push(filter); } - return filters; }; diff --git a/src/elastic-search/interfaces/es-common.type.ts b/src/elastic-search/interfaces/es-common.type.ts index 8cc66f227..8b49f5e01 100644 --- a/src/elastic-search/interfaces/es-common.type.ts +++ b/src/elastic-search/interfaces/es-common.type.ts @@ -20,6 +20,12 @@ export interface IShould { term?: { [key: string]: string | undefined; }; + range?: { + [key: string]: { + gte?: string | number; + lte?: string | number; + }; + }; } export interface IBoolShould { @@ -56,6 +62,10 @@ export interface IFilter { }; }; }; + bool?: { + should: IShould[]; + minimum_should_match?: number; + }; } export interface IFullFacets { @@ -81,3 +91,11 @@ export interface ITransformedFullFacets { } | { totalSets: number }; } + +export interface ScientificQuery { + [key: string]: Record | "$and" | "$or"; +} + +export interface nestedQueryObject { + [key: string]: Record; +} diff --git a/src/elastic-search/providers/query-builder.service.ts b/src/elastic-search/providers/query-builder.service.ts index 930f877f9..8d11fb6aa 100644 --- a/src/elastic-search/providers/query-builder.service.ts +++ b/src/elastic-search/providers/query-builder.service.ts @@ -7,6 +7,7 @@ import { IFullFacets, IShould, ObjectType, + ScientificQuery, } from "../interfaces/es-common.type"; import { FilterFields, @@ -125,7 +126,7 @@ export class SearchQueryService { ); const esScientificFilterQuery = convertToElasticSearchQuery( - scientificFilterQuery, + scientificFilterQuery as ScientificQuery, ); filterArray.push({ nested: { diff --git a/src/samples/samples.service.spec.ts b/src/samples/samples.service.spec.ts index 9425bec06..1933161b4 100644 --- a/src/samples/samples.service.spec.ts +++ b/src/samples/samples.service.spec.ts @@ -91,9 +91,23 @@ describe("SamplesService", () => { $text: { $search: "test", }, - "sampleCharacteristics.test.value": { - $eq: "test", - }, + $or: [ + { + "sampleCharacteristics.test": { + $eq: "test", + }, + }, + { + "sampleCharacteristics.test.v": { + $eq: "test", + }, + }, + { + "sampleCharacteristics.test.value": { + $eq: "test", + }, + }, + ], }; await service.fullquery(filter); diff --git a/test/DatasetFilter.js b/test/DatasetFilter.js index 37b71bf79..5dedb8e5a 100644 --- a/test/DatasetFilter.js +++ b/test/DatasetFilter.js @@ -62,6 +62,10 @@ const RawCorrect3 = { value: 6, unit: "", }, + test_field_string: { + value: "test_string_value", + unit: "", + }, }, datasetName: "This is the third correct test raw dataset", description: @@ -790,9 +794,9 @@ describe("0400: DatasetFilter: Test retrieving datasets using filtering capabili mode: {}, scientific: [ { - lhs: "test_field_1", + lhs: "test_field_string", relation: "EQUAL_TO_STRING", - rhs: "6", + rhs: "test_string_value", unit: "", }, ], @@ -808,7 +812,7 @@ describe("0400: DatasetFilter: Test retrieving datasets using filtering capabili .expect(TestData.SuccessfulGetStatusCode) .expect("Content-Type", /json/) .then((res) => { - res.body.length.should.be.equal(0); + res.body.length.should.be.equal(1); }); });