Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(search): Detect field type for use in defining the sort order #8992

Merged
merged 2 commits into from
Oct 18, 2023
Merged

fix(search): Detect field type for use in defining the sort order #8992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b837e8d
fix(search): Detect field type for use in defining the sort order
Oct 11, 2023
0222c44
Merge branch 'master' into nd--sort-unmapped-type
iprentic Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 20 additions & 28 deletions ...rc/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.linkedin.metadata.models.SearchScoreFieldSpec;
import com.linkedin.metadata.models.SearchableFieldSpec;
import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType;
import com.linkedin.metadata.search.utils.ESUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -31,15 +32,6 @@ public static Map<String, String> getPartialNgramConfigWithOverrides(Map<String,

public static final Map<String, String> KEYWORD_TYPE_MAP = ImmutableMap.of(TYPE, KEYWORD);

// Field Types
public static final String BOOLEAN = "boolean";
public static final String DATE = "date";
public static final String DOUBLE = "double";
public static final String LONG = "long";
public static final String OBJECT = "object";
public static final String TEXT = "text";
public static final String TOKEN_COUNT = "token_count";

// Subfields
public static final String DELIMITED = "delimited";
public static final String LENGTH = "length";
Expand Down Expand Up @@ -74,7 +66,7 @@ public static Map<String, Object> getMappings(@Nonnull final EntitySpec entitySp
private static Map<String, Object> getMappingsForUrn() {
Map<String, Object> subFields = new HashMap<>();
subFields.put(DELIMITED, ImmutableMap.of(
TYPE, TEXT,
TYPE, ESUtils.TEXT_FIELD_TYPE,
ANALYZER, URN_ANALYZER,
SEARCH_ANALYZER, URN_SEARCH_ANALYZER,
SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER)
Expand All @@ -85,13 +77,13 @@ private static Map<String, Object> getMappingsForUrn() {
)
));
return ImmutableMap.<String, Object>builder()
.put(TYPE, KEYWORD)
.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE)
.put(FIELDS, subFields)
.build();
}

private static Map<String, Object> getMappingsForRunId() {
return ImmutableMap.<String, Object>builder().put(TYPE, KEYWORD).build();
return ImmutableMap.<String, Object>builder().put(TYPE, ESUtils.KEYWORD_FIELD_TYPE).build();
}

private static Map<String, Object> getMappingsForField(@Nonnull final SearchableFieldSpec searchableFieldSpec) {
Expand All @@ -104,23 +96,23 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
} else if (fieldType == FieldType.TEXT || fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) {
mappingForField.putAll(getMappingsForSearchText(fieldType));
} else if (fieldType == FieldType.BROWSE_PATH) {
mappingForField.put(TYPE, TEXT);
mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
mappingForField.put(FIELDS,
ImmutableMap.of(LENGTH, ImmutableMap.of(
TYPE, TOKEN_COUNT,
TYPE, ESUtils.TOKEN_COUNT_FIELD_TYPE,
ANALYZER, SLASH_PATTERN_ANALYZER)));
mappingForField.put(ANALYZER, BROWSE_PATH_HIERARCHY_ANALYZER);
mappingForField.put(FIELDDATA, true);
} else if (fieldType == FieldType.BROWSE_PATH_V2) {
mappingForField.put(TYPE, TEXT);
mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
mappingForField.put(FIELDS,
ImmutableMap.of(LENGTH, ImmutableMap.of(
TYPE, TOKEN_COUNT,
TYPE, ESUtils.TOKEN_COUNT_FIELD_TYPE,
ANALYZER, UNIT_SEPARATOR_PATTERN_ANALYZER)));
mappingForField.put(ANALYZER, BROWSE_PATH_V2_HIERARCHY_ANALYZER);
mappingForField.put(FIELDDATA, true);
} else if (fieldType == FieldType.URN || fieldType == FieldType.URN_PARTIAL) {
mappingForField.put(TYPE, TEXT);
mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
mappingForField.put(ANALYZER, URN_ANALYZER);
mappingForField.put(SEARCH_ANALYZER, URN_SEARCH_ANALYZER);
mappingForField.put(SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER);
Expand All @@ -135,32 +127,32 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
subFields.put(KEYWORD, KEYWORD_TYPE_MAP);
mappingForField.put(FIELDS, subFields);
} else if (fieldType == FieldType.BOOLEAN) {
mappingForField.put(TYPE, BOOLEAN);
mappingForField.put(TYPE, ESUtils.BOOLEAN_FIELD_TYPE);
} else if (fieldType == FieldType.COUNT) {
mappingForField.put(TYPE, LONG);
mappingForField.put(TYPE, ESUtils.LONG_FIELD_TYPE);
} else if (fieldType == FieldType.DATETIME) {
mappingForField.put(TYPE, DATE);
mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
} else if (fieldType == FieldType.OBJECT) {
mappingForField.put(TYPE, OBJECT);
mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
} else {
log.info("FieldType {} has no mappings implemented", fieldType);
}
mappings.put(searchableFieldSpec.getSearchableAnnotation().getFieldName(), mappingForField);

searchableFieldSpec.getSearchableAnnotation()
.getHasValuesFieldName()
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, BOOLEAN)));
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.BOOLEAN_FIELD_TYPE)));
searchableFieldSpec.getSearchableAnnotation()
.getNumValuesFieldName()
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, LONG)));
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.LONG_FIELD_TYPE)));
mappings.putAll(getMappingsForFieldNameAliases(searchableFieldSpec));

return mappings;
}

private static Map<String, Object> getMappingsForKeyword() {
Map<String, Object> mappingForField = new HashMap<>();
mappingForField.put(TYPE, KEYWORD);
mappingForField.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE);
mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER);
// Add keyword subfield without lowercase filter
mappingForField.put(FIELDS, ImmutableMap.of(KEYWORD, KEYWORD_TYPE_MAP));
Expand All @@ -169,7 +161,7 @@ private static Map<String, Object> getMappingsForKeyword() {

private static Map<String, Object> getMappingsForSearchText(FieldType fieldType) {
Map<String, Object> mappingForField = new HashMap<>();
mappingForField.put(TYPE, KEYWORD);
mappingForField.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE);
mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER);
Map<String, Object> subFields = new HashMap<>();
if (fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) {
Expand All @@ -186,14 +178,14 @@ private static Map<String, Object> getMappingsForSearchText(FieldType fieldType)
String fieldName = entry.getKey();
String analyzerName = entry.getValue();
subFields.put(fieldName, ImmutableMap.of(
TYPE, TEXT,
TYPE, ESUtils.TEXT_FIELD_TYPE,
ANALYZER, analyzerName
));
}
}
}
subFields.put(DELIMITED, ImmutableMap.of(
TYPE, TEXT,
TYPE, ESUtils.TEXT_FIELD_TYPE,
ANALYZER, TEXT_ANALYZER,
SEARCH_ANALYZER, TEXT_SEARCH_ANALYZER,
SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER));
Expand All @@ -206,7 +198,7 @@ private static Map<String, Object> getMappingsForSearchText(FieldType fieldType)
private static Map<String, Object> getMappingsForSearchScoreField(
@Nonnull final SearchScoreFieldSpec searchScoreFieldSpec) {
return ImmutableMap.of(searchScoreFieldSpec.getSearchScoreAnnotation().getFieldName(),
ImmutableMap.of(TYPE, DOUBLE));
ImmutableMap.of(TYPE, ESUtils.DOUBLE_FIELD_TYPE));
}

private static Map<String, Object> getMappingsForFieldNameAliases(@Nonnull final SearchableFieldSpec searchableFieldSpec) {
Expand Down
8 changes: 4 additions & 4 deletions ...n/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
if (!finalSearchFlags.isSkipHighlighting()) {
searchSourceBuilder.highlighter(_highlights);
}
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);

if (finalSearchFlags.isGetSuggestions()) {
ESUtils.buildNameSuggestions(searchSourceBuilder, input);
Expand Down Expand Up @@ -243,7 +243,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).filter(filterQuery));
_aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation);
searchSourceBuilder.highlighter(getHighlights());
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
searchRequest.source(searchSourceBuilder);
log.debug("Search request is: " + searchRequest);
searchRequest.indicesOptions(null);
Expand All @@ -270,7 +270,7 @@ public SearchRequest getFilterRequest(@Nullable Filter filters, @Nullable SortCr
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(filterQuery);
searchSourceBuilder.from(from).size(size);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
searchRequest.source(searchSourceBuilder);

return searchRequest;
Expand Down Expand Up @@ -301,7 +301,7 @@ public SearchRequest getFilterRequest(@Nullable Filter filters, @Nullable SortCr
searchSourceBuilder.size(size);

ESUtils.setSearchAfter(searchSourceBuilder, sort, pitId, keepAlive);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
searchRequest.source(searchSourceBuilder);

return searchRequest;
Expand Down
74 changes: 71 additions & 3 deletions metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.linkedin.metadata.models.EntitySpec;
import com.linkedin.metadata.models.SearchableFieldSpec;
import com.linkedin.metadata.models.annotation.SearchableAnnotation;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.Criterion;
Expand Down Expand Up @@ -49,7 +52,28 @@ public class ESUtils {
public static final int MAX_RESULT_SIZE = 10000;
public static final String OPAQUE_ID_HEADER = "X-Opaque-Id";
public static final String HEADER_VALUE_DELIMITER = "|";
public static final String KEYWORD_TYPE = "keyword";

// Field types
public static final String KEYWORD_FIELD_TYPE = "keyword";
public static final String BOOLEAN_FIELD_TYPE = "boolean";
public static final String DATE_FIELD_TYPE = "date";
public static final String DOUBLE_FIELD_TYPE = "double";
public static final String LONG_FIELD_TYPE = "long";
public static final String OBJECT_FIELD_TYPE = "object";
public static final String TEXT_FIELD_TYPE = "text";
public static final String TOKEN_COUNT_FIELD_TYPE = "token_count";
// End of field types

public static final Set<SearchableAnnotation.FieldType> FIELD_TYPES_STORED_AS_KEYWORD = Set.of(
SearchableAnnotation.FieldType.KEYWORD,
SearchableAnnotation.FieldType.TEXT,
SearchableAnnotation.FieldType.TEXT_PARTIAL,
SearchableAnnotation.FieldType.WORD_GRAM);
public static final Set<SearchableAnnotation.FieldType> FIELD_TYPES_STORED_AS_TEXT = Set.of(
SearchableAnnotation.FieldType.BROWSE_PATH,
SearchableAnnotation.FieldType.BROWSE_PATH_V2,
SearchableAnnotation.FieldType.URN,
SearchableAnnotation.FieldType.URN_PARTIAL);
public static final String ENTITY_NAME_FIELD = "_entityName";
public static final String NAME_SUGGESTION = "nameSuggestion";

Expand Down Expand Up @@ -174,6 +198,25 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull final Criterion
return getQueryBuilderFromCriterionForSingleField(criterion, isTimeseries);
}

public static String getElasticTypeForFieldType(SearchableAnnotation.FieldType fieldType) {
if (FIELD_TYPES_STORED_AS_KEYWORD.contains(fieldType)) {
return KEYWORD_FIELD_TYPE;
} else if (FIELD_TYPES_STORED_AS_TEXT.contains(fieldType)) {
return TEXT_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.BOOLEAN) {
return BOOLEAN_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.COUNT) {
return LONG_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.DATETIME) {
return DATE_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.OBJECT) {
return OBJECT_FIELD_TYPE;
} else {
log.warn("FieldType {} has no mappings implemented", fieldType);
return null;
}
}

/**
* Populates source field of search query with the sort order as per the criterion provided.
*
Expand All @@ -189,14 +232,39 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull final Criterion
* @param sortCriterion {@link SortCriterion} to be applied to the search results
*/
public static void buildSortOrder(@Nonnull SearchSourceBuilder searchSourceBuilder,
@Nullable SortCriterion sortCriterion) {
@Nullable SortCriterion sortCriterion, List<EntitySpec> entitySpecs) {
if (sortCriterion == null) {
searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
} else {
Optional<SearchableAnnotation.FieldType> fieldTypeForDefault = Optional.empty();
for (EntitySpec entitySpec : entitySpecs) {
List<SearchableFieldSpec> fieldSpecs = entitySpec.getSearchableFieldSpecs();
for (SearchableFieldSpec fieldSpec : fieldSpecs) {
SearchableAnnotation annotation = fieldSpec.getSearchableAnnotation();
if (annotation.getFieldName().equals(sortCriterion.getField())
|| annotation.getFieldNameAliases().contains(sortCriterion.getField())) {
fieldTypeForDefault = Optional.of(fieldSpec.getSearchableAnnotation().getFieldType());
break;
}
}
if (fieldTypeForDefault.isPresent()) {
break;
}
}
if (fieldTypeForDefault.isEmpty()) {
log.warn("Sort criterion field " + sortCriterion.getField() + " was not found in any entity spec to be searched");
}
final SortOrder esSortOrder =
(sortCriterion.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING) ? SortOrder.ASC
: SortOrder.DESC;
searchSourceBuilder.sort(new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder).unmappedType(KEYWORD_TYPE));
FieldSortBuilder sortBuilder = new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder);
if (fieldTypeForDefault.isPresent()) {
String esFieldtype = getElasticTypeForFieldType(fieldTypeForDefault.get());
if (esFieldtype != null) {
sortBuilder.unmappedType(esFieldtype);
}
}
searchSourceBuilder.sort(sortBuilder);
}
if (sortCriterion == null || !sortCriterion.getField().equals(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD)) {
searchSourceBuilder.sort(new FieldSortBuilder(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD).order(SortOrder.ASC));
Expand Down
Loading
Loading