From 7a61f086cd1b762f32a4fa0faf237d50a9b3b897 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:28:03 -0800 Subject: [PATCH] add queryFieldNames field in Doc Level Queries (#582) (#597) --------- (cherry picked from commit 75925dcdbc98e29c07e007676ea6c68ee7468dec) Signed-off-by: Surya Sashank Nistala Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../commons/alerting/model/DocLevelQuery.kt | 71 +++++++++-- .../commons/alerting/model/WriteableTests.kt | 38 +++++- .../commons/alerting/model/XContentTests.kt | 119 ++++++++++++++++-- 3 files changed, 208 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/org/opensearch/commons/alerting/model/DocLevelQuery.kt b/src/main/kotlin/org/opensearch/commons/alerting/model/DocLevelQuery.kt index 5d3749dd..7c72b0ca 100644 --- a/src/main/kotlin/org/opensearch/commons/alerting/model/DocLevelQuery.kt +++ b/src/main/kotlin/org/opensearch/commons/alerting/model/DocLevelQuery.kt @@ -1,12 +1,12 @@ package org.opensearch.commons.alerting.model -import org.opensearch.common.io.stream.StreamInput -import org.opensearch.common.io.stream.StreamOutput -import org.opensearch.common.xcontent.XContentParserUtils import org.opensearch.commons.notifications.model.BaseModel +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.xcontent.ToXContent import org.opensearch.core.xcontent.XContentBuilder import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParserUtils import java.io.IOException import java.lang.IllegalArgumentException import java.util.UUID @@ -14,8 +14,10 @@ import java.util.UUID data class DocLevelQuery( val id: String = UUID.randomUUID().toString(), val name: String, + val fields: List, val query: String, - val tags: List = mutableListOf() + val tags: List = mutableListOf(), + val queryFieldNames: List = mutableListOf() ) : BaseModel { init { @@ -30,16 +32,20 @@ data class DocLevelQuery( constructor(sin: StreamInput) : this( sin.readString(), // id sin.readString(), // name + sin.readStringList(), // fields sin.readString(), // query - sin.readStringList() // tags + sin.readStringList(), // tags, + sin.readStringList() // fieldsBeingQueried ) fun asTemplateArg(): Map { return mapOf( QUERY_ID_FIELD to id, NAME_FIELD to name, + FIELDS_FIELD to fields, QUERY_FIELD to query, - TAGS_FIELD to tags + TAGS_FIELD to tags, + QUERY_FIELD_NAMES_FIELD to queryFieldNames ) } @@ -47,16 +53,20 @@ data class DocLevelQuery( override fun writeTo(out: StreamOutput) { out.writeString(id) out.writeString(name) + out.writeStringCollection(fields) out.writeString(query) out.writeStringCollection(tags) + out.writeStringCollection(queryFieldNames) } override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { builder.startObject() .field(QUERY_ID_FIELD, id) .field(NAME_FIELD, name) + .field(FIELDS_FIELD, fields.toTypedArray()) .field(QUERY_FIELD, query) .field(TAGS_FIELD, tags.toTypedArray()) + .field(QUERY_FIELD_NAMES_FIELD, queryFieldNames.toTypedArray()) .endObject() return builder } @@ -64,8 +74,10 @@ data class DocLevelQuery( companion object { const val QUERY_ID_FIELD = "id" const val NAME_FIELD = "name" + const val FIELDS_FIELD = "fields" const val QUERY_FIELD = "query" const val TAGS_FIELD = "tags" + const val QUERY_FIELD_NAMES_FIELD = "query_field_names" const val NO_ID = "" val INVALID_CHARACTERS: List = listOf(" ", "[", "]", "{", "}", "(", ")") @@ -76,6 +88,8 @@ data class DocLevelQuery( lateinit var query: String lateinit var name: String val tags: MutableList = mutableListOf() + val fields: MutableList = mutableListOf() + val queryFieldNames: MutableList = mutableListOf() XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { @@ -88,6 +102,7 @@ data class DocLevelQuery( name = xcp.text() validateQuery(name) } + QUERY_FIELD -> query = xcp.text() TAGS_FIELD -> { XContentParserUtils.ensureExpectedToken( @@ -101,14 +116,40 @@ data class DocLevelQuery( tags.add(tag) } } + + FIELDS_FIELD -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + xcp.currentToken(), + xcp + ) + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + val field = xcp.text() + fields.add(field) + } + } + + QUERY_FIELD_NAMES_FIELD -> { + XContentParserUtils.ensureExpectedToken( + XContentParser.Token.START_ARRAY, + xcp.currentToken(), + xcp + ) + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + val field = xcp.text() + queryFieldNames.add(field) + } + } } } return DocLevelQuery( id = id, name = name, + fields = fields, query = query, - tags = tags + tags = tags, + queryFieldNames = queryFieldNames ) } @@ -129,4 +170,20 @@ data class DocLevelQuery( } } } + + // constructor for java plugins' convenience to optionally avoid passing empty list for 'fieldsBeingQueried' field + constructor( + id: String, + name: String, + fields: MutableList, + query: String, + tags: MutableList + ) : this( + id = id, + name = name, + fields = fields, + query = query, + tags = tags, + queryFieldNames = emptyList() + ) } diff --git a/src/test/kotlin/org/opensearch/commons/alerting/model/WriteableTests.kt b/src/test/kotlin/org/opensearch/commons/alerting/model/WriteableTests.kt index 9f5e26b9..e81e59bd 100644 --- a/src/test/kotlin/org/opensearch/commons/alerting/model/WriteableTests.kt +++ b/src/test/kotlin/org/opensearch/commons/alerting/model/WriteableTests.kt @@ -3,13 +3,14 @@ package org.opensearch.commons.alerting.model import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.opensearch.common.io.stream.BytesStreamOutput -import org.opensearch.common.io.stream.StreamInput import org.opensearch.commons.alerting.model.action.Action import org.opensearch.commons.alerting.model.action.ActionExecutionPolicy import org.opensearch.commons.alerting.model.action.Throttle import org.opensearch.commons.alerting.randomAction import org.opensearch.commons.alerting.randomActionExecutionPolicy import org.opensearch.commons.alerting.randomBucketLevelTrigger +import org.opensearch.commons.alerting.randomChainedAlertTrigger +import org.opensearch.commons.alerting.randomDocLevelQuery import org.opensearch.commons.alerting.randomDocumentLevelTrigger import org.opensearch.commons.alerting.randomQueryLevelMonitor import org.opensearch.commons.alerting.randomQueryLevelTrigger @@ -17,7 +18,9 @@ import org.opensearch.commons.alerting.randomThrottle import org.opensearch.commons.alerting.randomUser import org.opensearch.commons.alerting.randomUserEmpty import org.opensearch.commons.authuser.User +import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.search.builder.SearchSourceBuilder +import kotlin.test.assertTrue class WriteableTests { @@ -111,6 +114,39 @@ class WriteableTests { Assertions.assertEquals(trigger, newTrigger, "Round tripping DocumentLevelTrigger doesn't work") } + @Test + fun `test doc-level query as stream`() { + val dlq = randomDocLevelQuery() + val out = BytesStreamOutput() + dlq.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newDlq = DocLevelQuery.readFrom(sin) + Assertions.assertEquals(dlq, newDlq, "Round tripping DocLevelQuery doesn't work") + assertTrue(newDlq.queryFieldNames.isEmpty()) + } + + @Test + fun `test doc-level query with query Field Names as stream`() { + val dlq = randomDocLevelQuery().copy(queryFieldNames = listOf("f1", "f2")) + val out = BytesStreamOutput() + dlq.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newDlq = DocLevelQuery.readFrom(sin) + assertTrue(newDlq.queryFieldNames.contains(dlq.queryFieldNames[0])) + assertTrue(newDlq.queryFieldNames.contains(dlq.queryFieldNames[1])) + Assertions.assertEquals(dlq, newDlq, "Round tripping DocLevelQuery doesn't work") + } + + @Test + fun `test chained alert trigger as stream`() { + val trigger = randomChainedAlertTrigger() + val out = BytesStreamOutput() + trigger.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newTrigger = ChainedAlertTrigger.readFrom(sin) + Assertions.assertEquals(trigger, newTrigger, "Round tripping DocumentLevelTrigger doesn't work") + } + @Test fun `test searchinput as stream`() { val input = SearchInput(emptyList(), SearchSourceBuilder()) diff --git a/src/test/kotlin/org/opensearch/commons/alerting/model/XContentTests.kt b/src/test/kotlin/org/opensearch/commons/alerting/model/XContentTests.kt index f8c98842..065191fb 100644 --- a/src/test/kotlin/org/opensearch/commons/alerting/model/XContentTests.kt +++ b/src/test/kotlin/org/opensearch/commons/alerting/model/XContentTests.kt @@ -22,6 +22,7 @@ import org.opensearch.commons.alerting.randomQueryLevelTrigger import org.opensearch.commons.alerting.randomThrottle import org.opensearch.commons.alerting.randomUser import org.opensearch.commons.alerting.randomUserEmpty +import org.opensearch.commons.alerting.randomWorkflow import org.opensearch.commons.alerting.toJsonString import org.opensearch.commons.alerting.toJsonStringWithUser import org.opensearch.commons.alerting.util.string @@ -95,7 +96,13 @@ class XContentTests { val throttleString = throttle.toXContent(builder(), ToXContent.EMPTY_PARAMS).string() val wrongThrottleString = throttleString.replace("MINUTES", "wrongunit") - assertFailsWith("Only support MINUTES throttle unit") { Throttle.parse(parser(wrongThrottleString)) } + assertFailsWith("Only support MINUTES throttle unit") { + Throttle.parse( + parser( + wrongThrottleString + ) + ) + } } @Test @@ -103,7 +110,13 @@ class XContentTests { val throttle = randomThrottle().copy(value = -1) val throttleString = throttle.toXContent(builder(), ToXContent.EMPTY_PARAMS).string() - assertFailsWith("Can only set positive throttle period") { Throttle.parse(parser(throttleString)) } + assertFailsWith("Can only set positive throttle period") { + Throttle.parse( + parser( + throttleString + ) + ) + } } fun `test query-level monitor parsing`() { @@ -131,7 +144,13 @@ class XContentTests { } """.trimIndent() - assertFailsWith("Monitor name is null") { Monitor.parse(parser(monitorStringWithoutName)) } + assertFailsWith("Monitor name is null") { + Monitor.parse( + parser( + monitorStringWithoutName + ) + ) + } } @Test @@ -160,6 +179,22 @@ class XContentTests { Assertions.assertEquals(monitor, parsedMonitor, "Round tripping BucketLevelMonitor doesn't work") } + @Test + fun `test composite workflow parsing`() { + val workflow = randomWorkflow() + val monitorString = workflow.toJsonStringWithUser() + val parsedMonitor = Workflow.parse(parser(monitorString)) + Assertions.assertEquals(workflow, parsedMonitor, "Round tripping BucketLevelMonitor doesn't work") + } + + @Test + fun `test composite workflow parsing with auditDelegateMonitorAlerts flag disabled`() { + val workflow = randomWorkflow(auditDelegateMonitorAlerts = false) + val monitorString = workflow.toJsonStringWithUser() + val parsedMonitor = Workflow.parse(parser(monitorString)) + Assertions.assertEquals(workflow, parsedMonitor, "Round tripping BucketLevelMonitor doesn't work") + } + @Test fun `test query-level trigger parsing`() { val trigger = randomQueryLevelTrigger() @@ -229,6 +264,32 @@ class XContentTests { Assertions.assertNull(parsedMonitor.user) } + @Test + fun `test workflow parsing`() { + val workflow = randomWorkflow(monitorIds = listOf("1", "2", "3")) + val monitorString = workflow.toJsonString() + val parsedWorkflow = Workflow.parse(parser(monitorString)) + Assertions.assertEquals(workflow, parsedWorkflow, "Round tripping workflow failed") + } + + @Test + fun `test chainedMonitorFindings parsing`() { + val cmf1 = ChainedMonitorFindings(monitorId = "m1") + val cmf1String = cmf1.toJsonString() + Assertions.assertEquals( + ChainedMonitorFindings.parse(parser(cmf1String)), + cmf1, + "Round tripping chained monitor findings failed" + ) + val cmf2 = ChainedMonitorFindings(monitorIds = listOf("m1", "m2")) + val cmf2String = cmf2.toJsonString() + Assertions.assertEquals( + ChainedMonitorFindings.parse(parser(cmf2String)), + cmf2, + "Round tripping chained monitor findings failed" + ) + } + @Test fun `test old monitor format parsing`() { val monitorString = """ @@ -358,6 +419,30 @@ class XContentTests { ) } + @Test + fun `test doc level query toXcontent`() { + val dlq = DocLevelQuery("id", "name", listOf("f1", "f2"), "query", listOf("t1", "t2")) + val dlqString = dlq.toXContent(builder(), ToXContent.EMPTY_PARAMS).string() + val parsedDlq = DocLevelQuery.parse(parser(dlqString)) + Assertions.assertEquals( + dlq, + parsedDlq, + "Round tripping Doc level query doesn't work" + ) + } + + @Test + fun `test doc level query toXcontent with query field names`() { + val dlq = DocLevelQuery("id", "name", listOf("f1", "f2"), "query", listOf("t1", "t2"), listOf("f1", "f2")) + val dlqString = dlq.toXContent(builder(), ToXContent.EMPTY_PARAMS).string() + val parsedDlq = DocLevelQuery.parse(parser(dlqString)) + Assertions.assertEquals( + dlq, + parsedDlq, + "Round tripping Doc level query doesn't work" + ) + } + @Test fun `test alert parsing`() { val alert = randomAlert() @@ -372,8 +457,15 @@ class XContentTests { fun `test alert parsing with noop trigger`() { val monitor = randomQueryLevelMonitor() val alert = Alert( - monitor = monitor, trigger = NoOpTrigger(), startTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), - errorMessage = "some error", lastNotificationTime = Instant.now() + id = "", + monitor = monitor, + trigger = NoOpTrigger(), + startTime = Instant.now().truncatedTo(ChronoUnit.MILLIS), + errorMessage = "some error", + lastNotificationTime = Instant.now(), + workflowId = "", + executionId = "", + clusters = listOf() ) assertEquals("Round tripping alert doesn't work", alert.triggerName, "NoOp trigger") } @@ -385,19 +477,22 @@ class XContentTests { "\"state\":\"ACTIVE\",\"error_message\":null,\"alert_history\":[],\"severity\":\"1\",\"action_execution_results\"" + ":[{\"action_id\":\"ghe1-XQBySl0wQKDBkOG\",\"last_execution_time\":1601917224583,\"throttled_count\":-1478015168}," + "{\"action_id\":\"gxe1-XQBySl0wQKDBkOH\",\"last_execution_time\":1601917224583,\"throttled_count\":-768533744}]," + - "\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null}" + "\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null," + + "\"clusters\":[\"cluster-1\",\"cluster-2\"]}" val parsedAlert = Alert.parse(parser(alertStr)) OpenSearchTestCase.assertNull(parsedAlert.monitorUser) } @Test fun `test alert parsing with user as null`() { - val alertStr = "{\"id\":\"\",\"version\":-1,\"monitor_id\":\"\",\"schema_version\":0,\"monitor_version\":1,\"monitor_user\":null," + - "\"monitor_name\":\"ARahqfRaJG\",\"trigger_id\":\"fhe1-XQBySl0wQKDBkOG\",\"trigger_name\":\"ffELMuhlro\"," + - "\"state\":\"ACTIVE\",\"error_message\":null,\"alert_history\":[],\"severity\":\"1\",\"action_execution_results\"" + - ":[{\"action_id\":\"ghe1-XQBySl0wQKDBkOG\",\"last_execution_time\":1601917224583,\"throttled_count\":-1478015168}," + - "{\"action_id\":\"gxe1-XQBySl0wQKDBkOH\",\"last_execution_time\":1601917224583,\"throttled_count\":-768533744}]," + - "\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null}" + val alertStr = + "{\"id\":\"\",\"version\":-1,\"monitor_id\":\"\",\"schema_version\":0,\"monitor_version\":1,\"monitor_user\":null," + + "\"monitor_name\":\"ARahqfRaJG\",\"trigger_id\":\"fhe1-XQBySl0wQKDBkOG\",\"trigger_name\":\"ffELMuhlro\"," + + "\"state\":\"ACTIVE\",\"error_message\":null,\"alert_history\":[],\"severity\":\"1\",\"action_execution_results\"" + + ":[{\"action_id\":\"ghe1-XQBySl0wQKDBkOG\",\"last_execution_time\":1601917224583,\"throttled_count\":-1478015168}," + + "{\"action_id\":\"gxe1-XQBySl0wQKDBkOH\",\"last_execution_time\":1601917224583,\"throttled_count\":-768533744}]," + + "\"start_time\":1601917224599,\"last_notification_time\":null,\"end_time\":null,\"acknowledged_time\":null," + + "\"clusters\":[\"cluster-1\",\"cluster-2\"]}" val parsedAlert = Alert.parse(parser(alertStr)) OpenSearchTestCase.assertNull(parsedAlert.monitorUser) }