From 1b87c3ca074ce81782fb902f69e541e63c5f8eaa Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Thu, 5 Oct 2023 19:59:15 +0200 Subject: [PATCH 1/5] Add sparse_vector field support --- .../kbn-field-utils/src/utils/field_types.ts | 1 + .../src/utils/get_field_type_description.ts | 4 ++++ .../src/utils/get_field_type_name.ts | 4 ++++ .../src/field_icon/field_icon.tsx | 2 ++ .../constants/data_types_definition.tsx | 17 +++++++++++++++++ .../mappings_editor/types/document_fields.ts | 1 + 6 files changed, 29 insertions(+) diff --git a/packages/kbn-field-utils/src/utils/field_types.ts b/packages/kbn-field-utils/src/utils/field_types.ts index 88d1a46949494..1d2649552479e 100644 --- a/packages/kbn-field-utils/src/utils/field_types.ts +++ b/packages/kbn-field-utils/src/utils/field_types.ts @@ -36,6 +36,7 @@ export enum KNOWN_FIELD_TYPES { RANK_FEATURES = 'rank_features', POINT = 'point', SHAPE = 'shape', + SPARSE_VECTOR = 'sparse_vector', STRING = 'string', TEXT = 'text', VERSION = 'version', diff --git a/packages/kbn-field-utils/src/utils/get_field_type_description.ts b/packages/kbn-field-utils/src/utils/get_field_type_description.ts index 3049da4304ec7..332f013b90db7 100644 --- a/packages/kbn-field-utils/src/utils/get_field_type_description.ts +++ b/packages/kbn-field-utils/src/utils/get_field_type_description.ts @@ -118,6 +118,10 @@ export function getFieldTypeDescription(type?: string) { return i18n.translate('fieldUtils.fieldNameDescription.shapeField', { defaultMessage: 'Arbitrary cartesian geometries.', }); + case KNOWN_FIELD_TYPES.SPARSE_VECTOR: + return i18n.translate('fieldUtils.fieldNameDescription.sparseVectorField', { + defaultMessage: 'Records sparse vectors of float values.', + }); case KNOWN_FIELD_TYPES.STRING: return i18n.translate('fieldUtils.fieldNameDescription.stringField', { defaultMessage: 'Full text such as the body of an email or a product description.', diff --git a/packages/kbn-field-utils/src/utils/get_field_type_name.ts b/packages/kbn-field-utils/src/utils/get_field_type_name.ts index 65ce720385b8e..bdf88897958b9 100644 --- a/packages/kbn-field-utils/src/utils/get_field_type_name.ts +++ b/packages/kbn-field-utils/src/utils/get_field_type_name.ts @@ -123,6 +123,10 @@ export function getFieldTypeName(type?: string) { return i18n.translate('fieldUtils.fieldNameIcons.shapeFieldAriaLabel', { defaultMessage: 'Shape', }); + case KNOWN_FIELD_TYPES.SPARSE_VECTOR: + return i18n.translate('fieldUtils.fieldNameIcons.sparseVectorFieldAriaLabel', { + defaultMessage: 'Sparse vector', + }); case KNOWN_FIELD_TYPES.STRING: return i18n.translate('fieldUtils.fieldNameIcons.stringFieldAriaLabel', { defaultMessage: 'String', diff --git a/packages/kbn-react-field/src/field_icon/field_icon.tsx b/packages/kbn-react-field/src/field_icon/field_icon.tsx index c0f39ec12dbb6..94b0bd7eceb75 100644 --- a/packages/kbn-react-field/src/field_icon/field_icon.tsx +++ b/packages/kbn-react-field/src/field_icon/field_icon.tsx @@ -32,6 +32,7 @@ export interface FieldIconProps extends Omit { | '_source' | 'point' | 'shape' + | 'sparse_vector' | 'string' | string | 'nested' @@ -69,6 +70,7 @@ export const typeToEuiIconMap: Partial> = { _source: { iconType: 'editorCodeBlock', color: 'gray' }, point: { iconType: 'tokenShape' }, // there is no separate icon for `point` yet shape: { iconType: 'tokenShape' }, + sparse_vector: { iconType: 'tokenRankFeatures' }, string: { iconType: 'tokenString' }, text: { iconType: 'tokenString' }, keyword: { iconType: 'tokenKeyword' }, diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx index 89043c9fd7e69..27d00ad6f5bf3 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx @@ -851,6 +851,23 @@ export const TYPE_DEFINITION: { [key in DataType]: DataTypeDefinition } = {

), }, + sparse_vector: { + label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.sparseVectorDescription', { + defaultMessage: 'Sparse vector', + }), + value: 'sparse_vector', + documentation: { + main: 'sparse-vector.html', + }, + description: () => ( +

+ +

+ ), + }, point: { label: i18n.translate('xpack.idxMgmt.mappingsEditor.dataType.pointDescription', { defaultMessage: 'Point', diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts b/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts index 5f6f4c6728be0..e62414f23e3f2 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/types/document_fields.ts @@ -58,6 +58,7 @@ export type MainType = | 'rank_features' | 'shape' | 'search_as_you_type' + | 'sparse_vector' | 'date' | 'date_nanos' | 'geo_point' From ae9df2ffc04b9614c8e17ad3bec078626650d39a Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Thu, 5 Oct 2023 19:59:37 +0200 Subject: [PATCH 2/5] ML pipelines create sparse_vector fields instead of rank_features --- .../ml_inference/update_ml_inference_mappings.test.ts | 4 ++-- .../pipelines/ml_inference/update_ml_inference_mappings.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts index 59699929ec313..bb9d7395c58e5 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.test.ts @@ -44,7 +44,7 @@ describe('updateMlInferenceMappings', () => { input_one_expanded: { properties: { predicted_value: { - type: 'rank_features', + type: 'sparse_vector', }, model_id: { type: 'keyword', @@ -54,7 +54,7 @@ describe('updateMlInferenceMappings', () => { input_two_expanded: { properties: { predicted_value: { - type: 'rank_features', + type: 'sparse_vector', }, model_id: { type: 'keyword', diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts index 3cc2ef3c2ed4f..fa8d06bde503f 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/update_ml_inference_mappings.ts @@ -77,7 +77,7 @@ const generateTextExpansionMappingProperties = (sourceFields: string[], targetFi type: 'keyword', }, predicted_value: { - type: 'rank_features', + type: 'sparse_vector', }, }, }, @@ -97,7 +97,7 @@ const formDefaultElserMappingProps = (sourceFields: string[]) => { type: 'keyword', }, predicted_value: { - type: 'rank_features', + type: 'sparse_vector', }, }, }, From d8086e8d1ee595987b09f8820abcb3f3cbdafeb0 Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Fri, 6 Oct 2023 10:40:46 +0200 Subject: [PATCH 3/5] dense_vector icon looks better on a sparse_vector field type --- packages/kbn-react-field/src/field_icon/field_icon.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-react-field/src/field_icon/field_icon.tsx b/packages/kbn-react-field/src/field_icon/field_icon.tsx index 94b0bd7eceb75..4db1c3a61663d 100644 --- a/packages/kbn-react-field/src/field_icon/field_icon.tsx +++ b/packages/kbn-react-field/src/field_icon/field_icon.tsx @@ -70,7 +70,7 @@ export const typeToEuiIconMap: Partial> = { _source: { iconType: 'editorCodeBlock', color: 'gray' }, point: { iconType: 'tokenShape' }, // there is no separate icon for `point` yet shape: { iconType: 'tokenShape' }, - sparse_vector: { iconType: 'tokenRankFeatures' }, + sparse_vector: { iconType: 'tokenDenseVector' }, string: { iconType: 'tokenString' }, text: { iconType: 'tokenString' }, keyword: { iconType: 'tokenKeyword' }, From 316ca694edd974b377d69947ccd08635bf3ead3d Mon Sep 17 00:00:00 2001 From: carlosdelest Date: Fri, 6 Oct 2023 11:59:30 +0200 Subject: [PATCH 4/5] Fix test snapshot --- .../field_icon/__snapshots__/field_icon.test.tsx.snap | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap b/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap index ada9724ca04e5..d33152540f7ea 100644 --- a/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap +++ b/packages/kbn-react-field/src/field_icon/__snapshots__/field_icon.test.tsx.snap @@ -275,6 +275,16 @@ exports[`FieldIcon renders known field types shape is rendered 1`] = ` /> `; +exports[`FieldIcon renders known field types sparse_vector is rendered 1`] = ` + +`; + exports[`FieldIcon renders known field types string is rendered 1`] = ` Date: Tue, 10 Oct 2023 19:59:06 +0200 Subject: [PATCH 5/5] Add sparse vector to mappings editor --- .../mappings_editor/constants/data_types_definition.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx index 27d00ad6f5bf3..8035b6af08b48 100644 --- a/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx +++ b/x-pack/plugins/index_management/public/application/components/mappings_editor/constants/data_types_definition.tsx @@ -975,6 +975,7 @@ export const MAIN_TYPES: MainType[] = [ 'rank_features', 'search_as_you_type', 'shape', + 'sparse_vector', 'text', 'token_count', 'histogram',