From 0a5e0c5766831a995e610352a9e40929c4dd1e9c Mon Sep 17 00:00:00 2001 From: SJ Date: Mon, 4 Apr 2022 16:21:54 +0530 Subject: [PATCH 1/9] fix(explore): order by on eds attributes --- .../service/explore/RequestHandler.java | 1 + .../entity/EntityServiceEntityFetcher.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java index deeffe82..07bfa76c 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java @@ -99,6 +99,7 @@ QueryRequest buildQueryRequest( // 3. Add GroupBy addGroupByExpressions(builder, request); + // TODO: Push group by down to QS // 4. If there's no Group By, Set Limit, Offset and Order By. // Otherwise, specify a large limit and track actual limit, offset and order by expression list // so we can compute diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java index 42c4b244..127b2aee 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.hypertrace.core.query.service.client.QueryServiceClient; import org.hypertrace.entity.query.service.client.EntityQueryServiceClient; import org.hypertrace.entity.query.service.v1.EntityQueryRequest; import org.hypertrace.entity.query.service.v1.Expression; @@ -17,8 +18,10 @@ import org.hypertrace.gateway.service.common.converters.EntityServiceAndGatewayServiceConverter; import org.hypertrace.gateway.service.common.util.AttributeMetadataUtil; import org.hypertrace.gateway.service.common.util.ExpressionReader; +import org.hypertrace.gateway.service.common.util.OrderByUtil; import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs; import org.hypertrace.gateway.service.explore.ExploreRequestContext; +import org.hypertrace.gateway.service.v1.common.OrderByExpression; import org.hypertrace.gateway.service.v1.explore.ExploreRequest; public class EntityServiceEntityFetcher { @@ -56,6 +59,18 @@ private EntityQueryRequest buildRequest( addGroupBys(exploreRequest, builder); addSelections(requestContext, exploreRequest, builder); + // TODO: Push group by down to EQS + // If there is a group by, specify a large limit and track actual limit, offset and order by + // expression list, so we can compute these once the we get the results. + if (requestContext.hasGroupBy()) { + // Will need to do the ordering, limit and offset ourselves after we get the group by results + builder.setLimit(QueryServiceClient.DEFAULT_QUERY_SERVICE_GROUP_BY_LIMIT); + requestContext.setOrderByExpressions(getRequestOrderByExpressions(exploreRequest)); + } else { + // No Group By + throw new RuntimeException("Entity request handler only supports group by requests"); + } + return builder.build(); } @@ -119,4 +134,9 @@ private Filter.Builder buildFilter( exploreRequest.getFilter())) .addAllChildFilter(entityIdsInFilter); } + + private List getRequestOrderByExpressions(ExploreRequest request) { + return OrderByUtil.matchOrderByExpressionsAliasToSelectionAlias( + request.getOrderByList(), request.getSelectionList(), request.getTimeAggregationList()); + } } From e3f553551ed79e254a58845ecc23267e20671d5e Mon Sep 17 00:00:00 2001 From: SJ Date: Mon, 25 Apr 2022 17:42:46 +0530 Subject: [PATCH 2/9] change the limit variable --- .../service/explore/entity/EntityServiceEntityFetcher.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java index 127b2aee..e6d5ed42 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java @@ -4,7 +4,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.hypertrace.core.query.service.client.QueryServiceClient; import org.hypertrace.entity.query.service.client.EntityQueryServiceClient; import org.hypertrace.entity.query.service.v1.EntityQueryRequest; import org.hypertrace.entity.query.service.v1.Expression; @@ -25,6 +24,8 @@ import org.hypertrace.gateway.service.v1.explore.ExploreRequest; public class EntityServiceEntityFetcher { + private static final int DEFAULT_ENTITY_SERVICE_GROUP_BY_LIMIT = 10000; + private final AttributeMetadataProvider attributeMetadataProvider; private final EntityIdColumnsConfigs entityIdColumnsConfigs; private final EntityQueryServiceClient entityQueryServiceClient; @@ -64,7 +65,7 @@ private EntityQueryRequest buildRequest( // expression list, so we can compute these once the we get the results. if (requestContext.hasGroupBy()) { // Will need to do the ordering, limit and offset ourselves after we get the group by results - builder.setLimit(QueryServiceClient.DEFAULT_QUERY_SERVICE_GROUP_BY_LIMIT); + builder.setLimit(DEFAULT_ENTITY_SERVICE_GROUP_BY_LIMIT); requestContext.setOrderByExpressions(getRequestOrderByExpressions(exploreRequest)); } else { // No Group By From 998534ab1a512a8e3ca72d16fd6fa28eff6d2123 Mon Sep 17 00:00:00 2001 From: SJ Date: Sun, 27 Aug 2023 15:20:10 +0530 Subject: [PATCH 3/9] spotless --- .../org/hypertrace/gateway/service/explore/ExploreService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java index 072754ed..36fef376 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java @@ -110,8 +110,7 @@ private boolean isContextAnEntityType(ExploreRequest request) { private IRequestHandler getRequestHandler( ExploreRequest request, Map attributeMetadataMap) { - if (isContextAnEntityType(request) - && !hasTimeAggregations(request)) { + if (isContextAnEntityType(request) && !hasTimeAggregations(request)) { ExpressionContext expressionContext = new ExpressionContext( attributeMetadataMap, From a5826ce62898b8f8c8b64108c807687370ceba01 Mon Sep 17 00:00:00 2001 From: SJ Date: Sun, 27 Aug 2023 16:02:30 +0530 Subject: [PATCH 4/9] fix vulnerabilities --- gateway-service-api/build.gradle.kts | 4 ++-- gateway-service-factory/build.gradle.kts | 2 +- gateway-service-impl/build.gradle.kts | 6 +++--- gateway-service/build.gradle.kts | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gateway-service-api/build.gradle.kts b/gateway-service-api/build.gradle.kts index cb4f8e1a..c7da3ef7 100644 --- a/gateway-service-api/build.gradle.kts +++ b/gateway-service-api/build.gradle.kts @@ -22,7 +22,7 @@ protobuf { // the identifier, which can be referred to in the "plugins" // container of the "generateProtoTasks" closure. id("grpc_java") { - artifact = "io.grpc:protoc-gen-grpc-java:1.56.0" + artifact = "io.grpc:protoc-gen-grpc-java:1.57.2" } } generateProtoTasks { @@ -44,7 +44,7 @@ sourceSets { } dependencies { - api(platform("io.grpc:grpc-bom:1.56.0")) + api(platform("io.grpc:grpc-bom:1.57.2")) api("io.grpc:grpc-protobuf") api("io.grpc:grpc-stub") api("javax.annotation:javax.annotation-api:1.3.2") diff --git a/gateway-service-factory/build.gradle.kts b/gateway-service-factory/build.gradle.kts index c606c510..001260a5 100644 --- a/gateway-service-factory/build.gradle.kts +++ b/gateway-service-factory/build.gradle.kts @@ -3,7 +3,7 @@ plugins { } dependencies { - api("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.52") + api("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.59") implementation(project(":gateway-service-impl")) } diff --git a/gateway-service-impl/build.gradle.kts b/gateway-service-impl/build.gradle.kts index 271579db..779dc80f 100644 --- a/gateway-service-impl/build.gradle.kts +++ b/gateway-service-impl/build.gradle.kts @@ -20,8 +20,8 @@ dependencies { implementation("org.hypertrace.entity.service:entity-service-client:0.8.56") implementation("org.hypertrace.entity.service:entity-service-api:0.8.56") - implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.12.0") - implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.12.0") + implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.12.2") + implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.12.2") implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.52") // Config @@ -41,5 +41,5 @@ dependencies { testImplementation("org.mockito:mockito-core:4.10.0") testImplementation("org.mockito:mockito-inline:4.10.0") testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.17.1") - testImplementation("io.grpc:grpc-netty:1.56.0") + testImplementation("io.grpc:grpc-netty:1.57.2") } diff --git a/gateway-service/build.gradle.kts b/gateway-service/build.gradle.kts index be980a15..3f036b93 100644 --- a/gateway-service/build.gradle.kts +++ b/gateway-service/build.gradle.kts @@ -8,12 +8,12 @@ plugins { dependencies { implementation(project(":gateway-service-factory")) - implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.12.0") - implementation("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.52") + implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.12.2") + implementation("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.59") implementation("org.slf4j:slf4j-api:1.7.30") implementation("com.typesafe:config:1.4.1") - runtimeOnly("io.grpc:grpc-netty:1.56.0") + runtimeOnly("io.grpc:grpc-netty:1.57.2") runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.17.1") } From b4b28519e66ed2b8d7c1f19f6a9dd7a5a9a01cc1 Mon Sep 17 00:00:00 2001 From: SJ Date: Fri, 13 Oct 2023 18:08:29 +0530 Subject: [PATCH 5/9] better comment --- .../service/explore/entity/EntityRequestHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java index 397e6d1f..0b2008f2 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java @@ -46,7 +46,7 @@ *
    *
  • when the selections, group bys and order bys are on EDS. A group by would need an attribute * selection, aggregation on the same attribute, and order by on any attribute - *
  • when selections, and order bys are on EDS. No group by would mean an aggregated selection + *
  • when aggregated selection is on EDS. No group by would mean a single aggregated selection * on attribute *
* @@ -60,8 +60,8 @@ *
  • Query to {@link * org.hypertrace.gateway.service.common.datafetcher.QueryServiceEntityFetcher} with the time * filter to get set of entity ids after applying QS filters - *
  • Query to {@link EntityServiceEntityFetcher} with selections, group bys(if present), and - * filters with an IN clause on entity ids + *
  • Query to {@link EntityServiceEntityFetcher} with selections(attribute + aggregated), group + * bys(if present), and filters with an IN clause on entity ids * */ public class EntityRequestHandler extends RequestHandler { From 7c6a52b73cf79179507031ebe973ebc93957d537 Mon Sep 17 00:00:00 2001 From: SJ Date: Fri, 13 Oct 2023 18:50:27 +0530 Subject: [PATCH 6/9] fix vulnerability --- owasp-suppressions.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/owasp-suppressions.xml b/owasp-suppressions.xml index 6f07f0cd..d3486420 100644 --- a/owasp-suppressions.xml +++ b/owasp-suppressions.xml @@ -17,4 +17,16 @@ ^pkg:maven/com\.fasterxml\.jackson\.core/jackson\-databind@.*$ CVE-2023-35116 + + + ^pkg:maven/io\.netty/netty.*@.*$ + CVE-2023-4586 + From 4c5f3570930b8a3118d553b186998a1121f971e8 Mon Sep 17 00:00:00 2001 From: SJ Date: Mon, 30 Oct 2023 14:21:42 +0530 Subject: [PATCH 7/9] push limit and offset to EQS --- .../entity/EntityServiceEntityFetcher.java | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java index 64ca8fe3..f169763b 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityServiceEntityFetcher.java @@ -23,8 +23,11 @@ import org.hypertrace.gateway.service.explore.ExploreRequestContext; import org.hypertrace.gateway.service.v1.common.OrderByExpression; import org.hypertrace.gateway.service.v1.explore.ExploreRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EntityServiceEntityFetcher { + private static final Logger LOG = LoggerFactory.getLogger(EntityServiceEntityFetcher.class); private static final int DEFAULT_ENTITY_REQUEST_LIMIT = 10_000; private final AttributeMetadataProvider attributeMetadataProvider; @@ -60,16 +63,19 @@ private EntityQueryRequest buildRequest( addGroupBys(exploreRequest, builder); addSelections(requestContext, exploreRequest, builder); - - // Ideally, needs the limit and offset for group by, since the fetcher is only triggered when - // there is a group by, or a single aggregation selection. A single aggregated selection would - // always return a single result (i.e. limit 1) - builder.setLimit(DEFAULT_ENTITY_REQUEST_LIMIT); - - // TODO: Push group by down to EQS - // If there is a group by, specify a large limit and track actual limit, offset and order by - // expression list, so we can compute these once the we get the results. - if (requestContext.hasGroupBy()) { + addLimitAndOffset(requestContext, exploreRequest, builder); + + // TODO: Push order by down to EQS + // EQS (and document-store) currently doesn't support order by on functional expressions + // If there are order by expressions, specify a large limit and track actual limit, offset and + // order by + // expression list, so we can compute these once we get the results. + if (!requestContext.getOrderByExpressions().isEmpty()) { + // Ideally, needs the limit and offset for group by, since the fetcher is only triggered when + // there is a group by, or a single aggregation selection. A single aggregated selection would + // always return a single result (i.e. limit 1) + builder.setOffset(0); + builder.setLimit(DEFAULT_ENTITY_REQUEST_LIMIT); // Will need to do the ordering, limit and offset ourselves after we get the group by results requestContext.setOrderByExpressions(getRequestOrderByExpressions(exploreRequest)); } @@ -107,6 +113,33 @@ private void addSelections( }); } + private void addLimitAndOffset( + ExploreRequestContext requestContext, + ExploreRequest exploreRequest, + EntityQueryRequest.Builder builder) { + // handle group by scenario with group limit set + if (requestContext.hasGroupBy()) { + int limit = exploreRequest.getLimit(); + if (exploreRequest.getGroupLimit() > 0) { + // in group by scenario, set limit to minimum of limit or group-limit + limit = Math.min(exploreRequest.getLimit(), exploreRequest.getGroupLimit()); + } + // don't exceed default group by limit + if (limit > DEFAULT_ENTITY_REQUEST_LIMIT) { + LOG.error( + "Trying to query for rows more than the default limit {} : {}", + DEFAULT_ENTITY_REQUEST_LIMIT, + exploreRequest); + throw new UnsupportedOperationException( + "Trying to query for rows more than the default limit " + exploreRequest); + } + builder.setLimit(limit); + } else { + builder.setLimit(exploreRequest.getLimit()); + builder.setOffset(exploreRequest.getOffset()); + } + } + private Filter.Builder buildFilter( ExploreRequest exploreRequest, List entityIdAttributeIds, Set entityIds) { Builder filterBuilder = From 9680686f7998be844caa1fe438e5f38819e19682 Mon Sep 17 00:00:00 2001 From: SJ Date: Mon, 30 Oct 2023 14:24:19 +0530 Subject: [PATCH 8/9] upgrade grpc service framework --- gateway-service-factory/build.gradle.kts | 2 +- gateway-service/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gateway-service-factory/build.gradle.kts b/gateway-service-factory/build.gradle.kts index 001260a5..5f106949 100644 --- a/gateway-service-factory/build.gradle.kts +++ b/gateway-service-factory/build.gradle.kts @@ -3,7 +3,7 @@ plugins { } dependencies { - api("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.59") + api("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.62") implementation(project(":gateway-service-impl")) } diff --git a/gateway-service/build.gradle.kts b/gateway-service/build.gradle.kts index aa74cf0c..5448a416 100644 --- a/gateway-service/build.gradle.kts +++ b/gateway-service/build.gradle.kts @@ -9,7 +9,7 @@ dependencies { implementation(project(":gateway-service-factory")) implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.12.5") - implementation("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.59") + implementation("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.62") implementation("org.slf4j:slf4j-api:1.7.30") implementation("com.typesafe:config:1.4.1") From 53df6eaa2efa399b06b1ba91af96eba2581d385b Mon Sep 17 00:00:00 2001 From: SJ Date: Mon, 30 Oct 2023 14:26:41 +0530 Subject: [PATCH 9/9] upgrade dependencies --- gateway-service-impl/build.gradle.kts | 16 ++++++++-------- gateway-service/build.gradle.kts | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gateway-service-impl/build.gradle.kts b/gateway-service-impl/build.gradle.kts index 71ece3b8..5b25ebe6 100644 --- a/gateway-service-impl/build.gradle.kts +++ b/gateway-service-impl/build.gradle.kts @@ -15,14 +15,14 @@ dependencies { annotationProcessor("org.projectlombok:lombok:1.18.22") compileOnly("org.projectlombok:lombok:1.18.18") - implementation("org.hypertrace.core.query.service:query-service-client:0.8.0") - implementation("org.hypertrace.core.attribute.service:attribute-service-client:0.14.25") - - implementation("org.hypertrace.entity.service:entity-service-client:0.8.56") - implementation("org.hypertrace.entity.service:entity-service-api:0.8.56") - implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.12.5") - implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.12.5") - implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.58") + implementation("org.hypertrace.core.query.service:query-service-client:0.8.20") + implementation("org.hypertrace.core.attribute.service:attribute-service-client:0.14.35") + + implementation("org.hypertrace.entity.service:entity-service-client:0.8.86") + implementation("org.hypertrace.entity.service:entity-service-api:0.8.86") + implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.12.6") + implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.12.6") + implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.62") // Config implementation("com.typesafe:config:1.4.1") diff --git a/gateway-service/build.gradle.kts b/gateway-service/build.gradle.kts index 5448a416..08e9f8f4 100644 --- a/gateway-service/build.gradle.kts +++ b/gateway-service/build.gradle.kts @@ -8,7 +8,7 @@ plugins { dependencies { implementation(project(":gateway-service-factory")) - implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.12.5") + implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.12.6") implementation("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.62") implementation("org.slf4j:slf4j-api:1.7.30") implementation("com.typesafe:config:1.4.1")