diff --git a/common/constants.ts b/common/constants.ts index 7c03329d..5acfc4d9 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -157,6 +157,10 @@ export const IMAGE_FIELD_PATTERN = `{{image_field}}`; export const QUERY_TEXT_PATTERN = `{{query_text}}`; export const QUERY_IMAGE_PATTERN = `{{query_image}}`; export const MODEL_ID_PATTERN = `{{model_id}}`; +export const VECTOR = 'vector'; +export const VECTOR_PATTERN = `{{${VECTOR}}}`; +export const VECTOR_TEMPLATE_PLACEHOLDER = `\$\{${VECTOR}\}`; +export const DEFAULT_K = 10; export const FETCH_ALL_QUERY = { query: { @@ -180,10 +184,9 @@ export const KNN_QUERY = { query: { knn: { [VECTOR_FIELD_PATTERN]: { - vector: `{{vector}}`, + vector: VECTOR_PATTERN, + k: DEFAULT_K, }, - k: 10, - model_id: MODEL_ID_PATTERN, }, }, }; @@ -196,7 +199,7 @@ export const SEMANTIC_SEARCH_QUERY_NEURAL = { [VECTOR_FIELD_PATTERN]: { query_text: QUERY_TEXT_PATTERN, model_id: MODEL_ID_PATTERN, - k: 100, + k: DEFAULT_K, }, }, }, @@ -211,7 +214,7 @@ export const MULTIMODAL_SEARCH_QUERY_NEURAL = { query_text: QUERY_TEXT_PATTERN, query_image: QUERY_IMAGE_PATTERN, model_id: MODEL_ID_PATTERN, - k: 100, + k: DEFAULT_K, }, }, }, @@ -234,6 +237,32 @@ export const MULTIMODAL_SEARCH_QUERY_BOOL = { }, }, }; +export const HYBRID_SEARCH_QUERY_MATCH_KNN = { + _source: { + excludes: [VECTOR_FIELD_PATTERN], + }, + query: { + hybrid: { + queries: [ + { + match: { + [TEXT_FIELD_PATTERN]: { + query: QUERY_TEXT_PATTERN, + }, + }, + }, + { + knn: { + [VECTOR_FIELD_PATTERN]: { + vector: VECTOR_PATTERN, + k: DEFAULT_K, + }, + }, + }, + ], + }, + }, +}; export const HYBRID_SEARCH_QUERY_MATCH_NEURAL = { _source: { excludes: [VECTOR_FIELD_PATTERN], @@ -253,7 +282,7 @@ export const HYBRID_SEARCH_QUERY_MATCH_NEURAL = { [VECTOR_FIELD_PATTERN]: { query_text: QUERY_TEXT_PATTERN, model_id: MODEL_ID_PATTERN, - k: 5, + k: DEFAULT_K, }, }, }, @@ -312,6 +341,10 @@ export const QUERY_PRESETS = [ name: `${WORKFLOW_TYPE.MULTIMODAL_SEARCH} (neural)`, query: customStringify(MULTIMODAL_SEARCH_QUERY_NEURAL), }, + { + name: `Hybrid search (match & k-NN queries)`, + query: customStringify(HYBRID_SEARCH_QUERY_MATCH_KNN), + }, { name: `Hybrid search (match & term queries)`, query: customStringify(HYBRID_SEARCH_QUERY_MATCH_TERM), diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx index 12adb49f..e4b65aa5 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx @@ -26,14 +26,18 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { // selected option state const [selectedOption, setSelectedOption] = useState([]); - // update the selected option when the form is updated. set to empty if the form value is undefined - // or an empty string ('') + // update the selected option when the form is updated. if the form is empty, + // default to the top option. by default, this will re-trigger this hook with a populated + // value, to then finally update the displayed option. useEffect(() => { const formValue = getIn(values, props.fieldPath); if (!isEmpty(formValue)) { setSelectedOption([{ label: getIn(values, props.fieldPath) }]); } else { - setSelectedOption([]); + if (props.options.length > 0) { + setFieldTouched(props.fieldPath, true); + setFieldValue(props.fieldPath, props.options[0].label); + } } }, [getIn(values, props.fieldPath)]); diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx index 528554eb..141df93f 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx @@ -12,10 +12,12 @@ import { EuiFlexGroup, EuiFlexItem, EuiFormRow, + EuiSmallButton, EuiSuperSelect, EuiSuperSelectOption, EuiText, EuiTitle, + EuiSpacer, } from '@elastic/eui'; import { SearchHit, WorkflowFormValues } from '../../../../../common'; import { JsonField } from '../input_fields'; @@ -142,41 +144,46 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) { /> - { - // for this test query, we don't want to involve any configured search pipelines, if any exist - // see https://opensearch.org/docs/latest/search-plugins/search-pipelines/using-search-pipeline/#disabling-the-default-pipeline-for-a-request - dispatch( - searchIndex({ - apiBody: { - index: values.search.index.name, - body: values.search.request, - searchPipeline: '_none', - }, - dataSourceId, - }) - ) - .unwrap() - .then(async (resp) => { - props.setQueryResponse( - JSON.stringify( - resp.hits.hits.map((hit: SearchHit) => hit._source), - undefined, - 2 - ) - ); - }) - .catch((error: any) => { - props.setQueryResponse(''); - console.error('Error running query: ', error); - }); - }} - > - Test - + <> + { + // for this test query, we don't want to involve any configured search pipelines, if any exist + // see https://opensearch.org/docs/latest/search-plugins/search-pipelines/using-search-pipeline/#disabling-the-default-pipeline-for-a-request + dispatch( + searchIndex({ + apiBody: { + index: values.search.index.name, + body: values.search.request, + searchPipeline: '_none', + }, + dataSourceId, + }) + ) + .unwrap() + .then(async (resp) => { + props.setQueryResponse( + JSON.stringify( + resp.hits.hits.map((hit: SearchHit) => hit._source), + undefined, + 2 + ) + ); + }) + .catch((error: any) => { + props.setQueryResponse(''); + console.error('Error running query: ', error); + }); + }} + > + Test + + + + Run query without any search pipeline configuration. + + diff --git a/public/pages/workflows/new_workflow/quick_configure_modal.tsx b/public/pages/workflows/new_workflow/quick_configure_modal.tsx index 73143778..b1edb744 100644 --- a/public/pages/workflows/new_workflow/quick_configure_modal.tsx +++ b/public/pages/workflows/new_workflow/quick_configure_modal.tsx @@ -24,6 +24,7 @@ import { MapFormValue, QuickConfigureFields, TEXT_FIELD_PATTERN, + VECTOR, VECTOR_FIELD_PATTERN, WORKFLOW_NAME_REGEXP, WORKFLOW_TYPE, @@ -172,8 +173,8 @@ function injectQuickConfigureFields( workflow.ui_metadata.config, quickConfigureFields ); - workflow.ui_metadata.config = updateSearchRequestConfig( - workflow.ui_metadata.config, + workflow.ui_metadata.config.search.request.value = injectPlaceholderValues( + (workflow.ui_metadata.config.search.request.value || '') as string, quickConfigureFields ); workflow.ui_metadata.config = updateSearchRequestProcessorConfig( @@ -228,6 +229,7 @@ function updateIngestProcessorConfig( } // prefill ML search request processor config, if applicable +// including populating placeholders in any pre-configured query_template function updateSearchRequestProcessorConfig( config: WorkflowConfig, fields: QuickConfigureFields @@ -236,10 +238,28 @@ function updateSearchRequestProcessorConfig( if (field.id === 'model' && fields.embeddingModelId) { field.value = { id: fields.embeddingModelId }; } - if (field.id === 'input_map' || field.id === 'output_map') { + if (field.id === 'input_map') { + // TODO: pre-populate more if the query becomes standard field.value = [[EMPTY_MAP_ENTRY]] as MapArrayFormValue; } + if (field.id === 'output_map') { + // prepopulate 'vector' constant as the model output transformed field, + // so it is consistent and used in the downstream query_template, if configured. + field.value = [[{ key: VECTOR, value: '' }]] as MapArrayFormValue; + } }); + config.search.enrichRequest.processors[0].optionalFields = config.search.enrichRequest.processors[0].optionalFields?.map( + (optionalField) => { + let updatedOptionalField = optionalField; + if (optionalField.id === 'query_template') { + optionalField.value = injectPlaceholderValues( + (optionalField.value || '') as string, + fields + ); + } + return updatedOptionalField; + } + ); return config; } @@ -295,39 +315,36 @@ function updateIndexConfig( return config; } -// pre-populate placeholders in the query, if applicable -function updateSearchRequestConfig( - config: WorkflowConfig, +// pre-populate placeholders for a query request string +function injectPlaceholderValues( + requestString: string, fields: QuickConfigureFields -): WorkflowConfig { +): string { + let finalRequestString = requestString; if (fields.embeddingModelId) { - config.search.request.value = ((config.search.request.value || - '') as string).replace( + finalRequestString = finalRequestString.replace( new RegExp(MODEL_ID_PATTERN, 'g'), fields.embeddingModelId ); } if (fields.textField) { - config.search.request.value = ((config.search.request.value || - '') as string).replace( + finalRequestString = finalRequestString.replace( new RegExp(TEXT_FIELD_PATTERN, 'g'), fields.textField ); } if (fields.vectorField) { - config.search.request.value = ((config.search.request.value || - '') as string).replace( + finalRequestString = finalRequestString.replace( new RegExp(VECTOR_FIELD_PATTERN, 'g'), fields.vectorField ); } if (fields.imageField) { - config.search.request.value = ((config.search.request.value || - '') as string).replace( + finalRequestString = finalRequestString.replace( new RegExp(IMAGE_FIELD_PATTERN, 'g'), fields.imageField ); } - return config; + return finalRequestString; } diff --git a/public/pages/workflows/new_workflow/utils.ts b/public/pages/workflows/new_workflow/utils.ts index d9652118..d41195b4 100644 --- a/public/pages/workflows/new_workflow/utils.ts +++ b/public/pages/workflows/new_workflow/utils.ts @@ -19,6 +19,11 @@ import { customStringify, TERM_QUERY, MULTIMODAL_SEARCH_QUERY_BOOL, + IProcessorConfig, + VECTOR_TEMPLATE_PLACEHOLDER, + VECTOR_PATTERN, + KNN_QUERY, + HYBRID_SEARCH_QUERY_MATCH_KNN, } from '../../../../common'; import { generateId } from '../../../utils'; @@ -124,13 +129,16 @@ function fetchSemanticSearchMetadata(): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.SEMANTIC_SEARCH; baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; - baseState.config.ingest.index.name.value = 'my-knn-index'; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); baseState.config.search.request.value = customStringify(TERM_QUERY); baseState.config.search.enrichRequest.processors = [ - new MLSearchRequestProcessor().toObj(), + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + KNN_QUERY + ), ]; return baseState; } @@ -139,7 +147,7 @@ function fetchMultimodalSearchMetadata(): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.MULTIMODAL_SEARCH; baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; - baseState.config.ingest.index.name.value = 'my-knn-index'; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); @@ -147,7 +155,10 @@ function fetchMultimodalSearchMetadata(): UIState { MULTIMODAL_SEARCH_QUERY_BOOL ); baseState.config.search.enrichRequest.processors = [ - new MLSearchRequestProcessor().toObj(), + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + MULTIMODAL_SEARCH_QUERY_BOOL + ), ]; return baseState; } @@ -156,7 +167,7 @@ function fetchHybridSearchMetadata(): UIState { let baseState = fetchEmptyMetadata(); baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH; baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()]; - baseState.config.ingest.index.name.value = 'my-knn-index'; + baseState.config.ingest.index.name.value = generateId('knn_index', 6); baseState.config.ingest.index.settings.value = customStringify({ [`index.knn`]: true, }); @@ -165,7 +176,10 @@ function fetchHybridSearchMetadata(): UIState { new NormalizationProcessor().toObj(), ]; baseState.config.search.enrichRequest.processors = [ - new MLSearchRequestProcessor().toObj(), + injectQueryTemplateInProcessor( + new MLSearchRequestProcessor().toObj(), + HYBRID_SEARCH_QUERY_MATCH_KNN + ), ]; return baseState; } @@ -178,3 +192,28 @@ export function processWorkflowName(workflowName: string): string { ? DEFAULT_NEW_WORKFLOW_NAME : snakeCase(workflowName); } + +// populate the `query_template` config value with a given query template +// by default, we replace any vector pattern ("{{vector}}") with the unquoted +// vector template placeholder (${vector}) so it becomes a proper template +function injectQueryTemplateInProcessor( + processorConfig: IProcessorConfig, + queryObj: {} +): IProcessorConfig { + processorConfig.optionalFields = processorConfig.optionalFields?.map( + (optionalField) => { + let updatedField = optionalField; + if (optionalField.id === 'query_template') { + updatedField = { + ...updatedField, + value: customStringify(queryObj).replace( + new RegExp(`"${VECTOR_PATTERN}"`, 'g'), + VECTOR_TEMPLATE_PLACEHOLDER + ), + }; + } + return updatedField; + } + ); + return processorConfig; +} diff --git a/public/utils/utils.ts b/public/utils/utils.ts index b22dffae..67d9ca88 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -30,15 +30,15 @@ import { DataSourceAttributes } from '../../../../src/plugins/data_source/common import { SavedObject } from '../../../../src/core/public'; import semver from 'semver'; -// Append 16 random characters -export function generateId(prefix?: string): string { +// Generate a random ID. Optionally add a prefix. Optionally +// override the default # characters to generate. +export function generateId(prefix?: string, numChars: number = 16): string { const uniqueChar = () => { // eslint-disable-next-line no-bitwise return (((1 + Math.random()) * 0x10000) | 0).toString(16).substring(1); }; - return `${ - prefix || '' - }_${uniqueChar()}${uniqueChar()}${uniqueChar()}${uniqueChar()}`; + const uniqueId = `${uniqueChar()}${uniqueChar()}${uniqueChar()}${uniqueChar()}`; + return `${prefix || ''}_${uniqueId.substring(0, numChars)}`; } export function sleep(ms: number) {