From fedaeffa005ab3382b905847f8462f482ca17a66 Mon Sep 17 00:00:00 2001 From: Ronak Date: Wed, 24 Aug 2022 17:14:08 +0530 Subject: [PATCH 1/6] fix: adding support for default group by --- .../documentstore/DocStoreQueryV1Test.java | 41 ++++++++++++++++++ .../query/v1/PostgresQueryParserTest.java | 43 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 5b5eba76..9133233a 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -44,7 +44,9 @@ import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; import org.hypertrace.core.documentstore.mongo.MongoDatastore; +import org.hypertrace.core.documentstore.postgres.Params; import org.hypertrace.core.documentstore.postgres.PostgresDatastore; +import org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser; import org.hypertrace.core.documentstore.query.Filter; import org.hypertrace.core.documentstore.query.Pagination; import org.hypertrace.core.documentstore.query.Query; @@ -54,7 +56,9 @@ import org.hypertrace.core.documentstore.query.SortingSpec; import org.hypertrace.core.documentstore.utils.Utils; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -708,6 +712,43 @@ public void testQueryQ1AggregationFilterWithStringInFilterAlongWithNonAliasField } } + @ParameterizedTest + @MethodSource("databaseContextBoth") + void testQueryQ1DistinctCountAggregationWithOnlyFilter(String dataStoreName) throws IOException { + Datastore datastore = datastoreMap.get(dataStoreName); + Collection collection = datastore.getCollection(COLLECTION_NAME); + org.hypertrace.core.documentstore.query.Query query = + org.hypertrace.core.documentstore.query.Query.builder() + .addSelection( + AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("quantity")), + "qty_count") + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("price")) + .setFilter( + LogicalExpression.builder() + .operator(AND) + .operand( + RelationalExpression.of( + IdentifierExpression.of("price"), LTE, ConstantExpression.of(10))) + .operand( + RelationalExpression.of( + IdentifierExpression.of("item"), + IN, + ConstantExpression.ofStrings( + List.of("Mirror", "Comb", "Shampoo", "Bottle")))) + .build()) + .build(); + + + try (CloseableIterator resultDocs = collection.aggregate(query)) { + Utils.assertDocsAndSizeEqualWithoutOrder( + dataStoreName, + resultDocs, + 3, + "mongo/test_string_in_filter_aggr_alias_distinct_count_response.json"); + } + } + @ParameterizedTest @MethodSource("databaseContextBoth") public void testQueryV1ForSimpleWhereClause(String dataStoreName) throws IOException { diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index d02da9a4..03a185e7 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -808,4 +808,47 @@ void testQueryQ1AggregationFilterWithStringInFilterAlongWithNonAliasFields() { Assertions.assertEquals("\"Shampoo\"", params.getObjectParams().get(4)); Assertions.assertEquals("\"Bottle\"", params.getObjectParams().get(5)); } + + @Test + void testQueryQ1DistinctCountAggregationWithOnlyFilter() { + org.hypertrace.core.documentstore.query.Query query = + org.hypertrace.core.documentstore.query.Query.builder() + .addSelection( + AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("quantity")), + "qty_count") + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("price")) + .setFilter( + LogicalExpression.builder() + .operator(AND) + .operand( + RelationalExpression.of( + IdentifierExpression.of("price"), LTE, ConstantExpression.of(10))) + .operand( + RelationalExpression.of( + IdentifierExpression.of("item"), + IN, + ConstantExpression.ofStrings( + List.of("Mirror", "Comb", "Shampoo", "Bottle")))) + .build()) + .build(); + + PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + String sql = postgresQueryParser.parse(); + + Assertions.assertEquals( + "SELECT COUNT(DISTINCT document->>'quantity' ) AS \"qty_count\", " + + "document->'item' AS item, document->'price' AS price " + + "FROM testCollection " + + "WHERE (CAST (document->>'price' AS NUMERIC) <= ?) AND (document->>'item' IN (?, ?, ?, ?))", + sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + Assertions.assertEquals(5, params.getObjectParams().size()); + Assertions.assertEquals(10, params.getObjectParams().get(1)); + Assertions.assertEquals("Mirror", params.getObjectParams().get(2)); + Assertions.assertEquals("Comb", params.getObjectParams().get(3)); + Assertions.assertEquals("Shampoo", params.getObjectParams().get(4)); + Assertions.assertEquals("Bottle", params.getObjectParams().get(5)); + } } From 4414858e4a41621af4d3fa636f4748c8eb20591e Mon Sep 17 00:00:00 2001 From: Ronak Date: Thu, 25 Aug 2022 16:10:30 +0530 Subject: [PATCH 2/6] trying out with different aggregation w/o group by --- .../documentstore/DocStoreQueryV1Test.java | 54 +++++++++++++---- .../query/v1/PostgresQueryParserTest.java | 60 +++++++++++++++++++ 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 92a8dc57..b6d3c11e 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -697,29 +697,59 @@ public void testQueryQ1AggregationFilterWithStringInFilterAlongWithNonAliasField void testQueryQ1DistinctCountAggregationWithOnlyFilter(String dataStoreName) throws IOException { Datastore datastore = datastoreMap.get(dataStoreName); Collection collection = datastore.getCollection(COLLECTION_NAME); +// org.hypertrace.core.documentstore.query.Query query = +// org.hypertrace.core.documentstore.query.Query.builder() +// .addSelection( +// AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("quantity")), +// "qty_count") +// .addSelection(IdentifierExpression.of("item")) +// .addSelection(IdentifierExpression.of("price")) +// .setFilter( +// LogicalExpression.builder() +// .operator(AND) +// .operand( +// RelationalExpression.of( +// IdentifierExpression.of("price"), LTE, ConstantExpression.of(10))) +// .operand( +// RelationalExpression.of( +// IdentifierExpression.of("item"), +// IN, +// ConstantExpression.ofStrings( +// List.of("Mirror", "Comb", "Shampoo", "Bottle")))) +// .build()) +// .build(); + org.hypertrace.core.documentstore.query.Query query = org.hypertrace.core.documentstore.query.Query.builder() .addSelection( - AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("quantity")), - "qty_count") - .addSelection(IdentifierExpression.of("item")) - .addSelection(IdentifierExpression.of("price")) + AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("attributes.service_id")), + "SERVICE_ID_DISTINCTCOUNT") + .addSelection( + AggregateExpression.of(SUM, IdentifierExpression.of("attributes.is_external")), + "SERVICE_IS_EXTERNAL") + .addSelection(IdentifierExpression.of("attributes.entity_id")) + .addSelection(IdentifierExpression.of("attributes.service_id")) .setFilter( LogicalExpression.builder() .operator(AND) .operand( RelationalExpression.of( - IdentifierExpression.of("price"), LTE, ConstantExpression.of(10))) - .operand( - RelationalExpression.of( - IdentifierExpression.of("item"), - IN, - ConstantExpression.ofStrings( - List.of("Mirror", "Comb", "Shampoo", "Bottle")))) + IdentifierExpression.of("attributes.is_external"), EQ, ConstantExpression.of(true))) + .operand(LogicalExpression.builder() + .operator(AND) + .operand(RelationalExpression.of( + IdentifierExpression.of("attributes.environment"), EQ, ConstantExpression.of("cluster001"))) + .operand(LogicalExpression.builder() + .operator(AND) + .operand(RelationalExpression.of( + IdentifierExpression.of("tenantId"), EQ, ConstantExpression.of("14d8d0d8-c1a9-4100-83a4-97edfeb85606"))) + .operand(RelationalExpression.of( + IdentifierExpression.of("type"), EQ, ConstantExpression.of("VULNERABILITY"))) + .build()) + .build()) .build()) .build(); - try (CloseableIterator resultDocs = collection.aggregate(query)) { Utils.assertDocsAndSizeEqualWithoutOrder( dataStoreName, diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index 03a185e7..c830b2c4 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -851,4 +851,64 @@ void testQueryQ1DistinctCountAggregationWithOnlyFilter() { Assertions.assertEquals("Shampoo", params.getObjectParams().get(4)); Assertions.assertEquals("Bottle", params.getObjectParams().get(5)); } + + /* + * SELECT COUNT(DISTINCT document->'attributes'->>'service_id' ) AS "SERVICE_ID_DISTINCTCOUNT", +document->'attributes'->'entity_id' AS "VULNERABILITY_FIELD_ENTITY_ID", +document->'attributes'->'service_id' AS "VULNERABILITY_FIELD_SERVICE_ID" +FROM insights +WHERE ((CAST (document->'attributes'->>'is_external' AS BOOLEAN) = 'TRUE') + AND (document->'attributes'->>'environment' = 'cluster001')) + AND (document->>'tenantId' = '14d8d0d8-c1a9-4100-83a4-97edfeb85606') + AND (document->>'type' = 'VULNERABILITY') +OFFSET 0 LIMIT 50 + * */ + @Test + void testQueryQ1DistinctCountAggregationWithOnlyFilterInshigts() { + org.hypertrace.core.documentstore.query.Query query = + org.hypertrace.core.documentstore.query.Query.builder() + .addSelection( + AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("attributes.service_id")), + "SERVICE_ID_DISTINCTCOUNT") + .addSelection(IdentifierExpression.of("attributes.entity_id")) + .addSelection(IdentifierExpression.of("attributes.service_id")) + .setFilter( + LogicalExpression.builder() + .operator(AND) + .operand( + RelationalExpression.of( + IdentifierExpression.of("attributes.is_external"), EQ, ConstantExpression.of(true))) + .operand(LogicalExpression.builder() + .operator(AND) + .operand(RelationalExpression.of( + IdentifierExpression.of("attributes.environment"), EQ, ConstantExpression.of("cluster001"))) + .operand(LogicalExpression.builder() + .operator(AND) + .operand(RelationalExpression.of( + IdentifierExpression.of("tenantId"), EQ, ConstantExpression.of("14d8d0d8-c1a9-4100-83a4-97edfeb85606"))) + .operand(RelationalExpression.of( + IdentifierExpression.of("type"), EQ, ConstantExpression.of("VULNERABILITY"))) + .build()) + .build()) + .build()) + .build(); + + PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + String sql = postgresQueryParser.parse(); + + Assertions.assertEquals( + "SELECT COUNT(DISTINCT document->>'quantity' ) AS \"qty_count\", " + + "document->'item' AS item, document->'price' AS price " + + "FROM testCollection " + + "WHERE (CAST (document->>'price' AS NUMERIC) <= ?) AND (document->>'item' IN (?, ?, ?, ?))", + sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + Assertions.assertEquals(5, params.getObjectParams().size()); + Assertions.assertEquals(10, params.getObjectParams().get(1)); + Assertions.assertEquals("Mirror", params.getObjectParams().get(2)); + Assertions.assertEquals("Comb", params.getObjectParams().get(3)); + Assertions.assertEquals("Shampoo", params.getObjectParams().get(4)); + Assertions.assertEquals("Bottle", params.getObjectParams().get(5)); + } } From 46d4491dd51423caee2972b46a914446e0be0a0e Mon Sep 17 00:00:00 2001 From: Ronak Date: Thu, 25 Aug 2022 16:16:49 +0530 Subject: [PATCH 3/6] adding distinct count query w/o group by --- .../documentstore/DocStoreQueryV1Test.java | 53 ++++--------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index b6d3c11e..796f9b2e 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -697,56 +697,25 @@ public void testQueryQ1AggregationFilterWithStringInFilterAlongWithNonAliasField void testQueryQ1DistinctCountAggregationWithOnlyFilter(String dataStoreName) throws IOException { Datastore datastore = datastoreMap.get(dataStoreName); Collection collection = datastore.getCollection(COLLECTION_NAME); -// org.hypertrace.core.documentstore.query.Query query = -// org.hypertrace.core.documentstore.query.Query.builder() -// .addSelection( -// AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("quantity")), -// "qty_count") -// .addSelection(IdentifierExpression.of("item")) -// .addSelection(IdentifierExpression.of("price")) -// .setFilter( -// LogicalExpression.builder() -// .operator(AND) -// .operand( -// RelationalExpression.of( -// IdentifierExpression.of("price"), LTE, ConstantExpression.of(10))) -// .operand( -// RelationalExpression.of( -// IdentifierExpression.of("item"), -// IN, -// ConstantExpression.ofStrings( -// List.of("Mirror", "Comb", "Shampoo", "Bottle")))) -// .build()) -// .build(); - org.hypertrace.core.documentstore.query.Query query = org.hypertrace.core.documentstore.query.Query.builder() .addSelection( - AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("attributes.service_id")), - "SERVICE_ID_DISTINCTCOUNT") - .addSelection( - AggregateExpression.of(SUM, IdentifierExpression.of("attributes.is_external")), - "SERVICE_IS_EXTERNAL") - .addSelection(IdentifierExpression.of("attributes.entity_id")) - .addSelection(IdentifierExpression.of("attributes.service_id")) + AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("quantity")), + "qty_count") + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("price")) .setFilter( LogicalExpression.builder() .operator(AND) .operand( RelationalExpression.of( - IdentifierExpression.of("attributes.is_external"), EQ, ConstantExpression.of(true))) - .operand(LogicalExpression.builder() - .operator(AND) - .operand(RelationalExpression.of( - IdentifierExpression.of("attributes.environment"), EQ, ConstantExpression.of("cluster001"))) - .operand(LogicalExpression.builder() - .operator(AND) - .operand(RelationalExpression.of( - IdentifierExpression.of("tenantId"), EQ, ConstantExpression.of("14d8d0d8-c1a9-4100-83a4-97edfeb85606"))) - .operand(RelationalExpression.of( - IdentifierExpression.of("type"), EQ, ConstantExpression.of("VULNERABILITY"))) - .build()) - .build()) + IdentifierExpression.of("price"), LTE, ConstantExpression.of(10))) + .operand( + RelationalExpression.of( + IdentifierExpression.of("item"), + IN, + ConstantExpression.ofStrings( + List.of("Mirror", "Comb", "Shampoo", "Bottle")))) .build()) .build(); From 98ba13524834e804344a432ce64b65254d502aab Mon Sep 17 00:00:00 2001 From: Ronak Date: Thu, 25 Aug 2022 21:11:18 +0530 Subject: [PATCH 4/6] feat: add query tranformer for handling selection --- .../documentstore/DocStoreQueryV1Test.java | 4 -- .../postgres/PostgresCollection.java | 11 +++ .../transformer/PostgresQueryTransformer.java | 25 +++++++ .../PostgresSelectionQueryTransformer.java | 67 +++++++++++++++++++ .../query/v1/PostgresQueryParserTest.java | 64 +++++++++++------- 5 files changed, 144 insertions(+), 27 deletions(-) create mode 100644 document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresQueryTransformer.java create mode 100644 document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 796f9b2e..93dc9b18 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -45,9 +45,7 @@ import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; import org.hypertrace.core.documentstore.mongo.MongoDatastore; -import org.hypertrace.core.documentstore.postgres.Params; import org.hypertrace.core.documentstore.postgres.PostgresDatastore; -import org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser; import org.hypertrace.core.documentstore.query.Filter; import org.hypertrace.core.documentstore.query.Pagination; import org.hypertrace.core.documentstore.query.Query; @@ -57,9 +55,7 @@ import org.hypertrace.core.documentstore.query.SortingSpec; import org.hypertrace.core.documentstore.utils.Utils; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java index be86284c..1df35e31 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java @@ -45,7 +45,9 @@ import org.hypertrace.core.documentstore.SingleValueKey; import org.hypertrace.core.documentstore.UpdateResult; import org.hypertrace.core.documentstore.commons.DocStoreConstants; +import org.hypertrace.core.documentstore.mongo.query.transformer.MongoQueryTransformer; import org.hypertrace.core.documentstore.postgres.internal.BulkUpdateSubDocsInternalResult; +import org.hypertrace.core.documentstore.postgres.query.v1.transformer.PostgresQueryTransformer; import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -779,6 +781,7 @@ private BulkUpdateResult upsertDocs(Map docs) throws IOException private CloseableIterator executeQueryV1( final org.hypertrace.core.documentstore.query.Query query) { + org.hypertrace.core.documentstore.query.Query transformedQuery = transformAndLog(query); org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser queryParser = new org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser( collectionName, query); @@ -800,6 +803,14 @@ private CloseableIterator executeQueryV1( } } + private org.hypertrace.core.documentstore.query.Query transformAndLog( + org.hypertrace.core.documentstore.query.Query query) { + LOGGER.debug("Original query before transformation: {}", query); + query = PostgresQueryTransformer.transform(query); + LOGGER.debug("Query after transformation: {}", query); + return query; + } + private boolean isValidType(Object v) { Set> validClassez = new HashSet<>() { diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresQueryTransformer.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresQueryTransformer.java new file mode 100644 index 00000000..0491bd2b --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresQueryTransformer.java @@ -0,0 +1,25 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.transformer; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.hypertrace.core.documentstore.query.Query; +import org.hypertrace.core.documentstore.query.transform.QueryTransformer; + +public class PostgresQueryTransformer { + + // Transform the query in the listed below order + private static final List TRANSFORMERS = + new ImmutableList.Builder() + .add(new PostgresSelectionQueryTransformer()) + .build(); + + public static Query transform(final Query query) { + Query transformedQuery = query; + + for (QueryTransformer transformer : TRANSFORMERS) { + transformedQuery = transformer.transform(transformedQuery); + } + + return transformedQuery; + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java new file mode 100644 index 00000000..6cdfe8e7 --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java @@ -0,0 +1,67 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.transformer; + +import java.util.List; +import java.util.stream.Collectors; +import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.parser.SelectTypeExpressionVisitor; +import org.hypertrace.core.documentstore.query.Query; +import org.hypertrace.core.documentstore.query.SelectionSpec; +import org.hypertrace.core.documentstore.query.transform.QueryTransformer; +import org.hypertrace.core.documentstore.query.transform.TransformedQueryBuilder; + +/* + * Postgres doesn't support the selection of attributes and aggregation w/o group by expression. + * e.g + * SELECT COUNT(DISTINCT document->>'quantity' ) AS QTY, document->'price' AS price + * FROM testCollection + * WHERE (CAST (document->>'price' AS NUMERIC) <= 10) + * + * So, if group by clause is missing, and selection contains any aggregation expression, + * this transformer removes all the non-aggregated expressions. So, the above query will be transformed + * to: + * + * SELECT COUNT(DISTINCT document->>'quantity' ) AS QTY + * FROM testCollection + * WHERE (CAST (document->>'price' AS NUMERIC) <= 10) + * + * This is the similar behavior supported in our other document store implementation (e.g Mongo) + * */ +public class PostgresSelectionQueryTransformer + implements QueryTransformer, SelectTypeExpressionVisitor { + + @Override + public Query transform(Query query) { + // no-op if group by clause exits + if (!query.getAggregations().isEmpty()) return query; + + // check for all selections, remove non-aggregated selections. + List finalSelectionSpecs = + query.getSelections().stream() + .filter(selectionSpec -> selectionSpec.getExpression().accept(this)) + .collect(Collectors.toUnmodifiableList()); + return new TransformedQueryBuilder(query).setSelections(finalSelectionSpecs).build(); + } + + @Override + public Boolean visit(AggregateExpression expression) { + return true; + } + + @Override + public Boolean visit(ConstantExpression expression) { + return false; + } + + @Override + public Boolean visit(FunctionExpression expression) { + return false; + } + + @Override + public Boolean visit(IdentifierExpression expression) { + return false; + } +} diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index c830b2c4..1081a621 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -30,6 +30,7 @@ import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; import org.hypertrace.core.documentstore.expression.operators.FunctionOperator; import org.hypertrace.core.documentstore.postgres.Params; +import org.hypertrace.core.documentstore.postgres.query.v1.transformer.PostgresQueryTransformer; import org.hypertrace.core.documentstore.query.Filter; import org.hypertrace.core.documentstore.query.Pagination; import org.hypertrace.core.documentstore.query.Query; @@ -833,7 +834,9 @@ void testQueryQ1DistinctCountAggregationWithOnlyFilter() { .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + Query transformedQuery = PostgresQueryTransformer.transform(query); + PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, transformedQuery); + String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -853,22 +856,23 @@ void testQueryQ1DistinctCountAggregationWithOnlyFilter() { } /* - * SELECT COUNT(DISTINCT document->'attributes'->>'service_id' ) AS "SERVICE_ID_DISTINCTCOUNT", -document->'attributes'->'entity_id' AS "VULNERABILITY_FIELD_ENTITY_ID", -document->'attributes'->'service_id' AS "VULNERABILITY_FIELD_SERVICE_ID" -FROM insights -WHERE ((CAST (document->'attributes'->>'is_external' AS BOOLEAN) = 'TRUE') - AND (document->'attributes'->>'environment' = 'cluster001')) - AND (document->>'tenantId' = '14d8d0d8-c1a9-4100-83a4-97edfeb85606') - AND (document->>'type' = 'VULNERABILITY') -OFFSET 0 LIMIT 50 - * */ + * SELECT COUNT(DISTINCT document->'attributes'->>'service_id' ) AS "SERVICE_ID_DISTINCTCOUNT", + document->'attributes'->'entity_id' AS "VULNERABILITY_FIELD_ENTITY_ID", + document->'attributes'->'service_id' AS "VULNERABILITY_FIELD_SERVICE_ID" + FROM insights + WHERE ((CAST (document->'attributes'->>'is_external' AS BOOLEAN) = 'TRUE') + AND (document->'attributes'->>'environment' = 'cluster001')) + AND (document->>'tenantId' = '14d8d0d8-c1a9-4100-83a4-97edfeb85606') + AND (document->>'type' = 'VULNERABILITY') + OFFSET 0 LIMIT 50 + * */ @Test void testQueryQ1DistinctCountAggregationWithOnlyFilterInshigts() { org.hypertrace.core.documentstore.query.Query query = org.hypertrace.core.documentstore.query.Query.builder() .addSelection( - AggregateExpression.of(DISTINCT_COUNT, IdentifierExpression.of("attributes.service_id")), + AggregateExpression.of( + DISTINCT_COUNT, IdentifierExpression.of("attributes.service_id")), "SERVICE_ID_DISTINCTCOUNT") .addSelection(IdentifierExpression.of("attributes.entity_id")) .addSelection(IdentifierExpression.of("attributes.service_id")) @@ -877,19 +881,33 @@ void testQueryQ1DistinctCountAggregationWithOnlyFilterInshigts() { .operator(AND) .operand( RelationalExpression.of( - IdentifierExpression.of("attributes.is_external"), EQ, ConstantExpression.of(true))) - .operand(LogicalExpression.builder() - .operator(AND) - .operand(RelationalExpression.of( - IdentifierExpression.of("attributes.environment"), EQ, ConstantExpression.of("cluster001"))) - .operand(LogicalExpression.builder() + IdentifierExpression.of("attributes.is_external"), + EQ, + ConstantExpression.of(true))) + .operand( + LogicalExpression.builder() .operator(AND) - .operand(RelationalExpression.of( - IdentifierExpression.of("tenantId"), EQ, ConstantExpression.of("14d8d0d8-c1a9-4100-83a4-97edfeb85606"))) - .operand(RelationalExpression.of( - IdentifierExpression.of("type"), EQ, ConstantExpression.of("VULNERABILITY"))) + .operand( + RelationalExpression.of( + IdentifierExpression.of("attributes.environment"), + EQ, + ConstantExpression.of("cluster001"))) + .operand( + LogicalExpression.builder() + .operator(AND) + .operand( + RelationalExpression.of( + IdentifierExpression.of("tenantId"), + EQ, + ConstantExpression.of( + "14d8d0d8-c1a9-4100-83a4-97edfeb85606"))) + .operand( + RelationalExpression.of( + IdentifierExpression.of("type"), + EQ, + ConstantExpression.of("VULNERABILITY"))) + .build()) .build()) - .build()) .build()) .build(); From e25529a3fed86e78dfd64c49b8805f4ef4520eed Mon Sep 17 00:00:00 2001 From: Ronak Date: Fri, 26 Aug 2022 10:48:29 +0530 Subject: [PATCH 5/6] add tests for corrosponding scenarios --- .../documentstore/DocStoreQueryV1Test.java | 4 +- .../test_aggr_only_with_fliter_response.json | 5 + .../postgres/PostgresCollection.java | 4 +- .../query/v1/PostgresQueryParser.java | 5 + .../PostgresSelectionQueryTransformer.java | 5 +- .../query/v1/PostgresQueryParserTest.java | 152 ++++++------------ 6 files changed, 66 insertions(+), 109 deletions(-) create mode 100644 document-store/src/integrationTest/resources/mongo/test_aggr_only_with_fliter_response.json diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 93dc9b18..c18a8032 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -719,8 +719,8 @@ void testQueryQ1DistinctCountAggregationWithOnlyFilter(String dataStoreName) thr Utils.assertDocsAndSizeEqualWithoutOrder( dataStoreName, resultDocs, - 3, - "mongo/test_string_in_filter_aggr_alias_distinct_count_response.json"); + 1, + "mongo/test_aggr_only_with_fliter_response.json"); } } diff --git a/document-store/src/integrationTest/resources/mongo/test_aggr_only_with_fliter_response.json b/document-store/src/integrationTest/resources/mongo/test_aggr_only_with_fliter_response.json new file mode 100644 index 00000000..4a9a6ffb --- /dev/null +++ b/document-store/src/integrationTest/resources/mongo/test_aggr_only_with_fliter_response.json @@ -0,0 +1,5 @@ +[ + { + "qty_count":3 + } +] \ No newline at end of file diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java index 1df35e31..09dd18fd 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/PostgresCollection.java @@ -45,7 +45,6 @@ import org.hypertrace.core.documentstore.SingleValueKey; import org.hypertrace.core.documentstore.UpdateResult; import org.hypertrace.core.documentstore.commons.DocStoreConstants; -import org.hypertrace.core.documentstore.mongo.query.transformer.MongoQueryTransformer; import org.hypertrace.core.documentstore.postgres.internal.BulkUpdateSubDocsInternalResult; import org.hypertrace.core.documentstore.postgres.query.v1.transformer.PostgresQueryTransformer; import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; @@ -781,10 +780,9 @@ private BulkUpdateResult upsertDocs(Map docs) throws IOException private CloseableIterator executeQueryV1( final org.hypertrace.core.documentstore.query.Query query) { - org.hypertrace.core.documentstore.query.Query transformedQuery = transformAndLog(query); org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser queryParser = new org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser( - collectionName, query); + collectionName, transformAndLog(query)); String sqlQuery = queryParser.parse(); try { PreparedStatement preparedStatement = diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParser.java index e4e8769e..adc3070d 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParser.java @@ -7,6 +7,7 @@ import lombok.Setter; import org.hypertrace.core.documentstore.postgres.Params; import org.hypertrace.core.documentstore.postgres.Params.Builder; +import org.hypertrace.core.documentstore.postgres.PostgresCollection; import org.hypertrace.core.documentstore.postgres.query.v1.transformer.FieldToPgColumnTransformer; import org.hypertrace.core.documentstore.postgres.query.v1.vistors.PostgresAggregationFilterTypeExpressionVisitor; import org.hypertrace.core.documentstore.postgres.query.v1.vistors.PostgresFilterTypeExpressionVisitor; @@ -17,8 +18,12 @@ import org.hypertrace.core.documentstore.postgres.query.v1.vistors.PostgresUnnestFilterTypeExpressionVisitor; import org.hypertrace.core.documentstore.query.Pagination; import org.hypertrace.core.documentstore.query.Query; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PostgresQueryParser { + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresCollection.class); + @Getter private final String collection; @Getter private final Query query; diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java index 6cdfe8e7..89a0589b 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java @@ -42,7 +42,10 @@ public Query transform(Query query) { query.getSelections().stream() .filter(selectionSpec -> selectionSpec.getExpression().accept(this)) .collect(Collectors.toUnmodifiableList()); - return new TransformedQueryBuilder(query).setSelections(finalSelectionSpecs).build(); + + return finalSelectionSpecs.size() > 0 + ? new TransformedQueryBuilder(query).setSelections(finalSelectionSpecs).build() + : query; } @Override diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index 1081a621..0f0a70ee 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -52,7 +52,8 @@ void testParseSimpleFilter() { RelationalExpression.of( IdentifierExpression.of("quantity"), NEQ, ConstantExpression.of(10))) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT * FROM testCollection " @@ -80,7 +81,8 @@ void testFilterWithNestedFiled() { ConstantExpression.of("Kolkata"))) .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT * FROM testCollection " @@ -108,7 +110,8 @@ void testFilterWithLogicalExpressionAnd() { IdentifierExpression.of("quantity"), LTE, ConstantExpression.of(10))) .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT * FROM testCollection WHERE (CAST (document->>'quantity' AS NUMERIC) >= ?) " @@ -135,7 +138,8 @@ void testFilterWithLogicalExpressionOr() { IdentifierExpression.of("quantity"), LTE, ConstantExpression.of(10))) .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT * FROM testCollection WHERE (CAST (document->>'quantity' AS NUMERIC) >= ?) " @@ -174,7 +178,8 @@ void testFilterWithLogicalExpressionAndOr() { .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT * FROM testCollection WHERE (CAST (document->>'price' AS NUMERIC) >= ?) " @@ -195,7 +200,8 @@ void testBasicSelectionExpression() { .addSelection(IdentifierExpression.of("item")) .addSelection(IdentifierExpression.of("price")) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT document->'item' AS item, document->'price' AS price FROM testCollection", sql); @@ -217,7 +223,8 @@ void testFunctionalSelectionExpression() { .build(), "total") .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT document->'item' AS item, " @@ -244,7 +251,8 @@ void testFunctionalSelectionExpressionWithNestedField() { .build(), "total") .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT document->'item' AS item, " @@ -273,7 +281,8 @@ void testFunctionalSelectionExpressionWithNestedFieldWithAlias() { .build(), "total") .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT document->'item' AS item, " @@ -311,7 +320,8 @@ void testAggregationExpression() { .addAggregation(IdentifierExpression.of("item")) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( "SELECT document->'item' AS item, " @@ -349,7 +359,8 @@ void testAggregationExpressionDistinctCount() { .addAggregation(IdentifierExpression.of("item")) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -385,7 +396,8 @@ void testAggregateWithMultipleGroupingLevels() { .addSort(IdentifierExpression.of("item"), DESC) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -415,7 +427,8 @@ void testAggregationFilter() { IdentifierExpression.of("qty_count"), LTE, ConstantExpression.of(10))) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -447,7 +460,8 @@ void testAggregationFilterAndWhereFilter() { IdentifierExpression.of("qty_count"), LTE, ConstantExpression.of(10))) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -487,7 +501,8 @@ void testAggregationFilterAlongWithNonAliasFields() { .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -515,7 +530,8 @@ void testSimpleOrderByClause() { .addSort(IdentifierExpression.of("item"), DESC) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -544,7 +560,8 @@ void testOrderByClauseWithAlias() { .addSort(IdentifierExpression.of("item"), DESC) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -596,7 +613,8 @@ void testFindWithSortingAndPagination() { .setPagination(pagination) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -632,7 +650,8 @@ void testUnnestWithoutPreserveNullAndEmptyArrays() { .addFromClause(UnnestExpression.of(IdentifierExpression.of("sales.medium"), false)) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -663,7 +682,8 @@ void testUnnestWithPreserveNullAndEmptyArrays() { .addFromClause(UnnestExpression.of(IdentifierExpression.of("sales.medium"), true)) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -705,7 +725,8 @@ void testUnnestWithoutPreserveNullAndEmptyArraysWithFilters() { .addFromClause(UnnestExpression.of(IdentifierExpression.of("sales.medium"), false)) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -749,7 +770,8 @@ void testQueryQ1AggregationFilterWithStringAlongWithNonAliasFields() { .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -791,7 +813,8 @@ void testQueryQ1AggregationFilterWithStringInFilterAlongWithNonAliasFields() { .build()) .build(); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( @@ -834,89 +857,12 @@ void testQueryQ1DistinctCountAggregationWithOnlyFilter() { .build()) .build(); - Query transformedQuery = PostgresQueryTransformer.transform(query); - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, transformedQuery); - - String sql = postgresQueryParser.parse(); - - Assertions.assertEquals( - "SELECT COUNT(DISTINCT document->>'quantity' ) AS \"qty_count\", " - + "document->'item' AS item, document->'price' AS price " - + "FROM testCollection " - + "WHERE (CAST (document->>'price' AS NUMERIC) <= ?) AND (document->>'item' IN (?, ?, ?, ?))", - sql); - - Params params = postgresQueryParser.getParamsBuilder().build(); - Assertions.assertEquals(5, params.getObjectParams().size()); - Assertions.assertEquals(10, params.getObjectParams().get(1)); - Assertions.assertEquals("Mirror", params.getObjectParams().get(2)); - Assertions.assertEquals("Comb", params.getObjectParams().get(3)); - Assertions.assertEquals("Shampoo", params.getObjectParams().get(4)); - Assertions.assertEquals("Bottle", params.getObjectParams().get(5)); - } - - /* - * SELECT COUNT(DISTINCT document->'attributes'->>'service_id' ) AS "SERVICE_ID_DISTINCTCOUNT", - document->'attributes'->'entity_id' AS "VULNERABILITY_FIELD_ENTITY_ID", - document->'attributes'->'service_id' AS "VULNERABILITY_FIELD_SERVICE_ID" - FROM insights - WHERE ((CAST (document->'attributes'->>'is_external' AS BOOLEAN) = 'TRUE') - AND (document->'attributes'->>'environment' = 'cluster001')) - AND (document->>'tenantId' = '14d8d0d8-c1a9-4100-83a4-97edfeb85606') - AND (document->>'type' = 'VULNERABILITY') - OFFSET 0 LIMIT 50 - * */ - @Test - void testQueryQ1DistinctCountAggregationWithOnlyFilterInshigts() { - org.hypertrace.core.documentstore.query.Query query = - org.hypertrace.core.documentstore.query.Query.builder() - .addSelection( - AggregateExpression.of( - DISTINCT_COUNT, IdentifierExpression.of("attributes.service_id")), - "SERVICE_ID_DISTINCTCOUNT") - .addSelection(IdentifierExpression.of("attributes.entity_id")) - .addSelection(IdentifierExpression.of("attributes.service_id")) - .setFilter( - LogicalExpression.builder() - .operator(AND) - .operand( - RelationalExpression.of( - IdentifierExpression.of("attributes.is_external"), - EQ, - ConstantExpression.of(true))) - .operand( - LogicalExpression.builder() - .operator(AND) - .operand( - RelationalExpression.of( - IdentifierExpression.of("attributes.environment"), - EQ, - ConstantExpression.of("cluster001"))) - .operand( - LogicalExpression.builder() - .operator(AND) - .operand( - RelationalExpression.of( - IdentifierExpression.of("tenantId"), - EQ, - ConstantExpression.of( - "14d8d0d8-c1a9-4100-83a4-97edfeb85606"))) - .operand( - RelationalExpression.of( - IdentifierExpression.of("type"), - EQ, - ConstantExpression.of("VULNERABILITY"))) - .build()) - .build()) - .build()) - .build(); - - PostgresQueryParser postgresQueryParser = new PostgresQueryParser(TEST_COLLECTION, query); + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser(TEST_COLLECTION, PostgresQueryTransformer.transform(query)); String sql = postgresQueryParser.parse(); Assertions.assertEquals( - "SELECT COUNT(DISTINCT document->>'quantity' ) AS \"qty_count\", " - + "document->'item' AS item, document->'price' AS price " + "SELECT COUNT(DISTINCT document->>'quantity' ) AS \"qty_count\" " + "FROM testCollection " + "WHERE (CAST (document->>'price' AS NUMERIC) <= ?) AND (document->>'item' IN (?, ?, ?, ?))", sql); From 066f8ee3c546c63a8d2edda98a5512b189d5c123 Mon Sep 17 00:00:00 2001 From: Ronak Date: Fri, 26 Aug 2022 12:48:20 +0530 Subject: [PATCH 6/6] applied spotless --- .../hypertrace/core/documentstore/DocStoreQueryV1Test.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index c18a8032..2a9adeb0 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -717,10 +717,7 @@ void testQueryQ1DistinctCountAggregationWithOnlyFilter(String dataStoreName) thr try (CloseableIterator resultDocs = collection.aggregate(query)) { Utils.assertDocsAndSizeEqualWithoutOrder( - dataStoreName, - resultDocs, - 1, - "mongo/test_aggr_only_with_fliter_response.json"); + dataStoreName, resultDocs, 1, "mongo/test_aggr_only_with_fliter_response.json"); } }