diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts index fa2e81ded897e..571e7c295e14c 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts @@ -91,14 +91,12 @@ describe('autocomplete.suggest', () => { test('on SPACE after "METADATA" keyword suggests all metadata fields', async () => { const { assertSuggestions } = await setup(); - await assertSuggestions('from a, b [METADATA /]', metadataFields); await assertSuggestions('from a, b METADATA /', metadataFields); }); test('on SPACE after "METADATA" column suggests command and pipe operators', async () => { const { assertSuggestions } = await setup(); - await assertSuggestions('from a, b [metadata _index /]', [',', '| ']); await assertSuggestions('from a, b metadata _index /', [',', '| ']); await assertSuggestions('from a, b metadata _index, _source /', [',', '| ']); await assertSuggestions(`from a, b metadata ${METADATA_FIELDS.join(', ')} /`, ['| ']); @@ -107,7 +105,6 @@ describe('autocomplete.suggest', () => { test('filters out already used metadata fields', async () => { const { assertSuggestions } = await setup(); - await assertSuggestions('from a, b [metadata _index, /]', metadataFieldsAndIndex); await assertSuggestions('from a, b metadata _index, /', metadataFieldsAndIndex); }); }); diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts index b89be15d670b1..f5a5e5ca551b7 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts @@ -104,7 +104,7 @@ describe('autocomplete', () => { .map(({ name }) => name.toUpperCase() + ' $0') ); testSuggestions( - 'from a [metadata _id] | /', + 'from a metadata _id | /', commandDefinitions .filter(({ name }) => !sourceCommands.includes(name)) .map(({ name }) => name.toUpperCase() + ' $0') @@ -116,7 +116,7 @@ describe('autocomplete', () => { .map(({ name }) => name.toUpperCase() + ' $0') ); testSuggestions( - 'from a [metadata _id] | eval var0 = a | /', + 'from a metadata _id | eval var0 = a | /', commandDefinitions .filter(({ name }) => !sourceCommands.includes(name)) .map(({ name }) => name.toUpperCase() + ' $0') diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/options.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/options.ts index 31d443a8cbb2b..87c91ddaabaa0 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/options.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/options.ts @@ -38,16 +38,6 @@ export const metadataOption: CommandOptionsDefinition = { skipCommonValidation: true, validate: (option, command, references) => { const messages: ESQLMessage[] = []; - // need to test the parent command here - if (/\[metadata/i.test(command.text)) { - messages.push( - getMessageFromId({ - messageId: 'metadataBracketsDeprecation', - values: {}, - locations: option.location, - }) - ); - } const fields = option.args.filter(isColumnItem); const metadataFieldsAvailable = references as unknown as Set; if (metadataFieldsAvailable.size > 0) { diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/test_suites/validation.command.from.ts b/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/test_suites/validation.command.from.ts index 491c44fe699df..2a316cdb7a2e9 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/test_suites/validation.command.from.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/test_suites/validation.command.from.ts @@ -106,7 +106,7 @@ export const validationFromCommandTestSuite = (setup: helpers.Setup) => { await expectErrors('from index metadata _id, \t\n _index\n ', []); }); - test('errors when wrapped in brackets', async () => { + test('errors when wrapped in parentheses', async () => { const { expectErrors } = await setup(); await expectErrors(`from index (metadata _id)`, [ @@ -114,65 +114,20 @@ export const validationFromCommandTestSuite = (setup: helpers.Setup) => { ]); }); - for (const isWrapped of [true, false]) { - function setWrapping(option: string) { - return isWrapped ? `[${option}]` : option; - } - - function addBracketsWarning() { - return isWrapped - ? ["Square brackets '[]' need to be removed from FROM METADATA declaration"] - : []; - } - - describe(`wrapped = ${isWrapped}`, () => { - test('no errors on correct usage, waning on square brackets', async () => { - const { expectErrors } = await setup(); - - await expectErrors(`from index ${setWrapping('METADATA _id')}`, []); - await expectErrors( - `from index ${setWrapping('METADATA _id')}`, - [], - addBracketsWarning() - ); - await expectErrors( - `from index ${setWrapping('metadata _id')}`, - [], - addBracketsWarning() - ); - await expectErrors( - `from index ${setWrapping('METADATA _id, _source')}`, - [], - addBracketsWarning() - ); - }); - - test('validates fields', async () => { - const { expectErrors } = await setup(); - - await expectErrors( - `from index ${setWrapping('METADATA _id, _source2')}`, - [ - `Metadata field [_source2] is not available. Available metadata fields are: [${METADATA_FIELDS.join( - ', ' - )}]`, - ], - addBracketsWarning() - ); - await expectErrors( - `from index ${setWrapping('metadata _id, _source')} ${setWrapping( - 'METADATA _id2' - )}`, - [ - isWrapped - ? "SyntaxError: mismatched input '[' expecting " - : "SyntaxError: mismatched input 'METADATA' expecting ", - ], - addBracketsWarning() - ); - }); + describe('validates fields', () => { + test('validates fields', async () => { + const { expectErrors } = await setup(); + + await expectErrors(`from index METADATA _id, _source2`, [ + `Metadata field [_source2] is not available. Available metadata fields are: [${METADATA_FIELDS.join( + ', ' + )}]`, + ]); + await expectErrors(`from index metadata _id, _source METADATA _id2`, [ + "SyntaxError: mismatched input 'METADATA' expecting ", + ]); }); - } + }); }); }); }); diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.ccs.test.ts b/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.ccs.test.ts index 5cdb83be618d1..81e54b0b5cf1a 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.ccs.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/__tests__/validation.ccs.test.ts @@ -24,33 +24,13 @@ describe('validation', () => { }); describe('... METADATA ', () => { - for (const isWrapped of [true, false]) { - function setWrapping(option: string) { - return isWrapped ? `[${option}]` : option; - } - - function addBracketsWarning() { - return isWrapped - ? ["Square brackets '[]' need to be removed from FROM METADATA declaration"] - : []; - } - - describe(`wrapped = ${isWrapped}`, () => { - test('no errors on correct usage, waning on square brackets', async () => { - const { expectErrors } = await setup(); - await expectErrors( - `from remote-ccs:indexes ${setWrapping('METADATA _id')}`, - ['Unknown index [remote-ccs:indexes]'], - addBracketsWarning() - ); - await expectErrors( - `from *:indexes ${setWrapping('METADATA _id')}`, - ['Unknown index [*:indexes]'], - addBracketsWarning() - ); - }); - }); - } + test('no errors on correct usage', async () => { + const { expectErrors } = await setup(); + await expectErrors(`from remote-ccs:indexes METADATA _id`, [ + 'Unknown index [remote-ccs:indexes]', + ]); + await expectErrors(`from *:indexes METADATA _id`, ['Unknown index [*:indexes]']); + }); }); }); }); diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json b/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json index f1e71c9ff6a97..25fb880da5ff0 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json +++ b/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json @@ -9947,70 +9947,6 @@ ], "warning": [] }, - { - "query": "from index [METADATA _id]", - "error": [], - "warning": [] - }, - { - "query": "from index [METADATA _id]", - "error": [], - "warning": [ - "Square brackets '[]' need to be removed from FROM METADATA declaration" - ] - }, - { - "query": "from index [metadata _id]", - "error": [], - "warning": [ - "Square brackets '[]' need to be removed from FROM METADATA declaration" - ] - }, - { - "query": "from index [METADATA _id, _source]", - "error": [], - "warning": [ - "Square brackets '[]' need to be removed from FROM METADATA declaration" - ] - }, - { - "query": "from index [METADATA _id, _source2]", - "error": [ - "Metadata field [_source2] is not available. Available metadata fields are: [_version, _id, _index, _source, _ignored, _index_mode]" - ], - "warning": [ - "Square brackets '[]' need to be removed from FROM METADATA declaration" - ] - }, - { - "query": "from index [metadata _id, _source] [METADATA _id2]", - "error": [ - "SyntaxError: mismatched input '[' expecting " - ], - "warning": [ - "Square brackets '[]' need to be removed from FROM METADATA declaration" - ] - }, - { - "query": "from index METADATA _id", - "error": [], - "warning": [] - }, - { - "query": "from index METADATA _id", - "error": [], - "warning": [] - }, - { - "query": "from index metadata _id", - "error": [], - "warning": [] - }, - { - "query": "from index METADATA _id, _source", - "error": [], - "warning": [] - }, { "query": "from index METADATA _id, _source2", "error": [ @@ -10019,11 +9955,11 @@ "warning": [] }, { - "query": "from index metadata _id, _source METADATA _id2", + "query": "from index METADATA _id, _source METADATA _id2", "error": [ "SyntaxError: mismatched input 'METADATA' expecting " ], "warning": [] } ] -} \ No newline at end of file +} diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts b/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts index a9ecac9663609..e6546ec996547 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts @@ -1741,7 +1741,7 @@ describe('validation logic', () => { it('should basically work when all callbacks are passed', async () => { const allErrors = await Promise.all( fixtures.testCases - .filter(({ query }) => query === 'from index [METADATA _id, _source2]') + .filter(({ query }) => query === 'from index METADATA _id, _source2') .map(({ query }) => validateQuery( query, @@ -1753,7 +1753,7 @@ describe('validation logic', () => { ); for (const [index, { errors }] of Object.entries(allErrors)) { expect(errors.map((e) => ('severity' in e ? e.message : e.text))).toEqual( - fixtures.testCases.filter(({ query }) => query === 'from index [METADATA _id, _source2]')[ + fixtures.testCases.filter(({ query }) => query === 'from index METADATA _id, _source2')[ Number(index) ].error ); diff --git a/packages/kbn-language-documentation/src/sections/esql_documentation_sections.tsx b/packages/kbn-language-documentation/src/sections/esql_documentation_sections.tsx index 66c93fc0f7787..59f9b999c8a5f 100644 --- a/packages/kbn-language-documentation/src/sections/esql_documentation_sections.tsx +++ b/packages/kbn-language-documentation/src/sections/esql_documentation_sections.tsx @@ -19,7 +19,7 @@ export const initialSection = ( @@ -77,7 +77,7 @@ ES|QL can access the following metadata fields: Use the \`METADATA\` directive to enable metadata fields: \`\`\` -FROM index [METADATA _index, _id] +FROM index METADATA _index, _id \`\`\` Metadata fields are only available if the source of the data is an index. Consequently, \`FROM\` is the only source commands that supports the \`METADATA\` directive. @@ -85,7 +85,7 @@ Metadata fields are only available if the source of the data is an index. Conseq Once enabled, the fields are then available to subsequent processing commands, just like the other index fields: \`\`\` -FROM ul_logs, apps [METADATA _index, _version] +FROM ul_logs, apps METADATA _index, _version | WHERE id IN (13, 14) AND _version == 1 | EVAL key = CONCAT(_index, "_", TO_STR(id)) | SORT id, _index @@ -95,7 +95,7 @@ FROM ul_logs, apps [METADATA _index, _version] Also, similar to the index fields, once an aggregation is performed, a metadata field will no longer be accessible to subsequent commands, unless used as grouping field: \`\`\` -FROM employees [METADATA _index, _id] +FROM employees METADATA _index, _id | STATS max = MAX(emp_no) BY _index \`\`\` `, @@ -114,7 +114,7 @@ FROM employees [METADATA _index, _id] markdownContent={i18n.translate('languageDocumentation.documentationESQL.row.markdown', { defaultMessage: `### ROW The \`ROW\` source command produces a row with one or more columns with values that you specify. This can be useful for testing. - + \`\`\` ROW a = 1, b = "two", c = null \`\`\` @@ -207,7 +207,7 @@ ROW a = "1953-01-23T12:15:00Z - some text - 127.0.0.1" markdownContent={i18n.translate('languageDocumentation.documentationESQL.drop.markdown', { defaultMessage: `### DROP Use \`DROP\` to remove columns from a table: - + \`\`\` FROM employees | DROP height @@ -347,7 +347,7 @@ ROW a = "12 15.5 15.6 true" The \`KEEP\` command enables you to specify what columns are returned and the order in which they are returned. To limit the columns that are returned, use a comma-separated list of column names. The columns are returned in the specified order: - + \`\`\` FROM employees | KEEP first_name, last_name, height @@ -384,7 +384,7 @@ FROM employees { defaultMessage: `### LIMIT The \`LIMIT\` processing command enables you to limit the number of rows: - + \`\`\` FROM employees | LIMIT 5 @@ -407,7 +407,7 @@ FROM employees 'languageDocumentation.documentationESQL.mvExpand.markdown', { defaultMessage: `### MV_EXPAND -The \`MV_EXPAND\` processing command expands multivalued fields into one row per value, duplicating other fields: +The \`MV_EXPAND\` processing command expands multivalued fields into one row per value, duplicating other fields: \`\`\` ROW a=[1,2,3], b="b", j=["a","b"] | MV_EXPAND a @@ -607,7 +607,7 @@ FROM employees { defaultMessage: `### WHERE Use \`WHERE\` to produce a table that contains all the rows from the input table for which the provided condition evaluates to \`true\`: - + \`\`\` FROM employees | KEEP first_name, last_name, still_hired @@ -654,7 +654,7 @@ export const groupingFunctions = { defaultMessage: `### BUCKET Creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range. -\`BUCKET\` works in two modes: +\`BUCKET\` works in two modes: 1. Where the size of the bucket is computed based on a buckets count recommendation (four parameters) and a range. 2. Where the bucket size is provided directly (two parameters). diff --git a/test/functional/apps/discover/group6/_sidebar_field_stats.ts b/test/functional/apps/discover/group6/_sidebar_field_stats.ts index 3cfa2c1da20af..325adb313ed6c 100644 --- a/test/functional/apps/discover/group6/_sidebar_field_stats.ts +++ b/test/functional/apps/discover/group6/_sidebar_field_stats.ts @@ -148,7 +148,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await discover.selectTextBaseLang(); - const testQuery = `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500`; + const testQuery = `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500`; await monacoEditor.setCodeEditorValue(testQuery); await testSubjects.click('querySubmitButton'); await header.waitUntilLoadingHasFinished(); @@ -168,7 +168,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await unifiedFieldList.clickFieldListPlusFilter('bytes', '0'); const editorValue = await monacoEditor.getCodeEditorValue(); expect(editorValue).to.eql( - `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`bytes\`==0` + `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`bytes\`==0` ); await unifiedFieldList.closeFieldPopover(); }); @@ -188,7 +188,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await unifiedFieldList.clickFieldListPlusFilter('extension.raw', 'css'); const editorValue = await monacoEditor.getCodeEditorValue(); expect(editorValue).to.eql( - `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`extension.raw\`=="css"` + `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`extension.raw\`=="css"` ); await unifiedFieldList.closeFieldPopover(); @@ -209,7 +209,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await unifiedFieldList.clickFieldListPlusFilter('clientip', '216.126.255.31'); const editorValue = await monacoEditor.getCodeEditorValue(); expect(editorValue).to.eql( - `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`clientip\`::string=="216.126.255.31"` + `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`clientip\`::string=="216.126.255.31"` ); await unifiedFieldList.closeFieldPopover(); @@ -234,7 +234,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await unifiedFieldList.clickFieldListExistsFilter('@timestamp'); const editorValue = await monacoEditor.getCodeEditorValue(); expect(editorValue).to.eql( - `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`@timestamp\` is not null` + `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`@timestamp\` is not null` ); await testSubjects.missingOrFail('dscFieldStats-statsFooter'); await unifiedFieldList.closeFieldPopover(); @@ -269,7 +269,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await unifiedFieldList.clickFieldListPlusFilter('extension', 'css'); const editorValue = await monacoEditor.getCodeEditorValue(); expect(editorValue).to.eql( - `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`extension\`=="css"` + `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`extension\`=="css"` ); await unifiedFieldList.closeFieldPopover(); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts index 9dcca93d8fdd2..808597ff36495 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts @@ -22,11 +22,11 @@ const getQeryAst = (query: string) => { describe('computeHasMetadataOperator', () => { it('should be false if query does not have operator', () => { expect(computeHasMetadataOperator(getQeryAst('from test*'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* [metadata]'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* [metadata id]'))).toBe(false); + expect(computeHasMetadataOperator(getQeryAst('from test* metadata'))).toBe(false); + expect(computeHasMetadataOperator(getQeryAst('from test* metadata id'))).toBe(false); expect(computeHasMetadataOperator(getQeryAst('from metadata*'))).toBe(false); expect(computeHasMetadataOperator(getQeryAst('from test* | keep metadata'))).toBe(false); - expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="[metadata _id]"'))).toBe( + expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="metadata _id"'))).toBe( false ); }); @@ -48,19 +48,19 @@ describe('computeHasMetadataOperator', () => { ).toBe(true); // still validates deprecated square bracket syntax - expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _id]'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _id, _index]'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _index, _id]'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id ]'))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id] '))).toBe(true); - expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id] | limit 10'))).toBe( + expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id'))).toBe(true); + expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id, _index'))).toBe(true); + expect(computeHasMetadataOperator(getQeryAst('from test* metadata _index, _id'))).toBe(true); + expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); + expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true); + expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id | limit 10'))).toBe( true ); expect( computeHasMetadataOperator( - getQeryAst(`from packetbeat* [metadata + getQeryAst(`from packetbeat* metadata - _id ] + _id | limit 100`) ) ).toBe(true); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts index 869e379c21aed..c508676cae92c 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts @@ -154,7 +154,7 @@ export const parseEsqlQuery = (query: string) => { return { errors, isEsqlQueryAggregating, - // non-aggregating query which does not have [metadata], is not a valid one + // non-aggregating query which does not have metadata, is not a valid one isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(ast), }; };