Skip to content
This repository has been archived by the owner on Jun 26, 2024. It is now read-only.

perf(entities): optimise AND filter query #193

Merged
merged 4 commits into from
Feb 28, 2024
Merged

perf(entities): optimise AND filter query #193

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a700ec7
perf(entities): optimise AND filter query
skjindal93 Feb 21, 2024
151d11b
remove time tracking logs
skjindal93 Feb 21, 2024
bfa41b5
fix and filter for filter optimizing visitor
skjindal93 Feb 26, 2024
271eab2
fix tests
skjindal93 Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 7 additions & 14 deletions gateway-service-impl/src/main/java/org/hypertrace/gateway/service/GatewayServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,11 @@
import org.hypertrace.gateway.service.baseline.BaselineServiceQueryParser;
import org.hypertrace.gateway.service.common.AttributeMetadataProvider;
import org.hypertrace.gateway.service.common.RequestContext;
import org.hypertrace.gateway.service.common.config.ScopeFilterConfigs;
import org.hypertrace.gateway.service.common.config.GatewayServiceConfig;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceClient;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceV2Client;
import org.hypertrace.gateway.service.common.util.QueryServiceClient;
import org.hypertrace.gateway.service.entity.EntityService;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.entity.config.LogConfig;
import org.hypertrace.gateway.service.executor.QueryExecutorConfig;
import org.hypertrace.gateway.service.executor.QueryExecutorServiceFactory;
import org.hypertrace.gateway.service.explore.ExploreService;
Expand Down Expand Up @@ -91,7 +89,6 @@ public GatewayServiceImpl(Config appConfig, GrpcChannelRegistry grpcChannelRegis
new AttributeServiceClient(
grpcChannelRegistry.forPlaintextAddress(asConfig.getHost(), asConfig.getPort()));
AttributeMetadataProvider attributeMetadataProvider = new AttributeMetadataProvider(asClient);
EntityIdColumnsConfigs entityIdColumnsConfigs = EntityIdColumnsConfigs.fromConfig(appConfig);

Config untypedQsConfig = appConfig.getConfig(QUERY_SERVICE_CONFIG_KEY);
QueryServiceConfig qsConfig = new QueryServiceConfig(untypedQsConfig);
Expand Down Expand Up @@ -124,41 +121,37 @@ public GatewayServiceImpl(Config appConfig, GrpcChannelRegistry grpcChannelRegis
EntityTypesProvider entityTypesProvider =
new EntityTypesProvider(entityTypeServiceClient, entityTypeServiceV2Client);

ScopeFilterConfigs scopeFilterConfigs = new ScopeFilterConfigs(appConfig);
LogConfig logConfig = new LogConfig(appConfig);
GatewayServiceConfig gatewayServiceConfig = new GatewayServiceConfig(appConfig);
this.traceService =
new TracesService(
queryServiceClient, attributeMetadataProvider, scopeFilterConfigs, queryExecutor);
gatewayServiceConfig, queryServiceClient, attributeMetadataProvider, queryExecutor);
this.spanService =
new SpanService(queryServiceClient, attributeMetadataProvider, queryExecutor);
this.entityService =
new EntityService(
gatewayServiceConfig,
queryServiceClient,
eqsClient,
eqsStub,
attributeMetadataProvider,
entityIdColumnsConfigs,
scopeFilterConfigs,
logConfig,
queryExecutor);
this.exploreService =
new ExploreService(
gatewayServiceConfig,
queryServiceClient,
eqsClient,
attributeMetadataProvider,
scopeFilterConfigs,
entityIdColumnsConfigs,
entityTypesProvider);
BaselineServiceQueryParser baselineServiceQueryParser =
new BaselineServiceQueryParser(attributeMetadataProvider);
BaselineServiceQueryExecutor baselineServiceQueryExecutor =
new BaselineServiceQueryExecutor(queryServiceClient);
this.baselineService =
new BaselineServiceImpl(
gatewayServiceConfig,
attributeMetadataProvider,
baselineServiceQueryParser,
baselineServiceQueryExecutor,
entityIdColumnsConfigs);
baselineServiceQueryExecutor);
this.logEventsService = new LogEventsService(queryServiceClient, attributeMetadataProvider);
initMetrics();
}
Expand Down
18 changes: 9 additions & 9 deletions ...rvice-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
import org.hypertrace.gateway.service.baseline.lib.BaselineCalculator;
import org.hypertrace.gateway.service.common.AttributeMetadataProvider;
import org.hypertrace.gateway.service.common.RequestContext;
import org.hypertrace.gateway.service.common.config.GatewayServiceConfig;
import org.hypertrace.gateway.service.common.util.AttributeMetadataUtil;
import org.hypertrace.gateway.service.common.util.QueryExpressionUtil;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.v1.baseline.Baseline;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse;
Expand All @@ -40,25 +40,25 @@
*/
public class BaselineServiceImpl implements BaselineService {

private static final BaselineEntitiesRequestValidator baselineEntitiesRequestValidator =
private static final BaselineEntitiesRequestValidator BASELINE_ENTITIES_REQUEST_VALIDATOR =
new BaselineEntitiesRequestValidator();

protected static final long DAY_IN_MILLIS = 86400000L;

private final GatewayServiceConfig gatewayServiceConfig;
private final AttributeMetadataProvider attributeMetadataProvider;
private final BaselineServiceQueryParser baselineServiceQueryParser;
private final BaselineServiceQueryExecutor baselineServiceQueryExecutor;
private final EntityIdColumnsConfigs entityIdColumnsConfigs;

public BaselineServiceImpl(
GatewayServiceConfig gatewayServiceConfig,
AttributeMetadataProvider attributeMetadataProvider,
BaselineServiceQueryParser baselineServiceQueryParser,
BaselineServiceQueryExecutor baselineServiceQueryExecutor,
EntityIdColumnsConfigs entityIdColumnsConfigs) {
BaselineServiceQueryExecutor baselineServiceQueryExecutor) {
this.gatewayServiceConfig = gatewayServiceConfig;
this.attributeMetadataProvider = attributeMetadataProvider;
this.baselineServiceQueryParser = baselineServiceQueryParser;
this.baselineServiceQueryExecutor = baselineServiceQueryExecutor;
this.entityIdColumnsConfigs = entityIdColumnsConfigs;
}

public BaselineEntitiesResponse getBaselineForEntities(
Expand All @@ -67,7 +67,7 @@ public BaselineEntitiesResponse getBaselineForEntities(
Map<String, AttributeMetadata> attributeMetadataMap =
attributeMetadataProvider.getAttributesMetadata(
requestContext, originalRequest.getEntityType());
baselineEntitiesRequestValidator.validate(originalRequest, attributeMetadataMap);
BASELINE_ENTITIES_REQUEST_VALIDATOR.validate(originalRequest, attributeMetadataMap);
String timeColumn =
AttributeMetadataUtil.getTimestampAttributeId(
attributeMetadataProvider, requestContext, originalRequest.getEntityType());
Expand All @@ -91,7 +91,7 @@ public BaselineEntitiesResponse getBaselineForEntities(
List<String> entityIdAttributes =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
gatewayServiceConfig.getEntityIdColumnsConfig(),
requestContext,
originalRequest.getEntityType());
QueryRequest aggQueryRequest =
Expand Down Expand Up @@ -134,7 +134,7 @@ public BaselineEntitiesResponse getBaselineForEntities(
List<String> entityIdAttributes =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
gatewayServiceConfig.getEntityIdColumnsConfig(),
requestContext,
originalRequest.getEntityType());
QueryRequest timeSeriesQueryRequest =
Expand Down
31 changes: 31 additions & 0 deletions ...y-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
import java.util.stream.Stream;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

logic class

import org.hypertrace.core.attribute.service.v1.AttributeMetadata;
import org.hypertrace.core.attribute.service.v1.AttributeSource;
import org.hypertrace.gateway.service.common.config.GatewayServiceConfig;
import org.hypertrace.gateway.service.common.util.ExpressionReader;
import org.hypertrace.gateway.service.common.util.OrderByUtil;
import org.hypertrace.gateway.service.v1.common.Expression;
import org.hypertrace.gateway.service.v1.common.Filter;
import org.hypertrace.gateway.service.v1.common.Operator;
import org.hypertrace.gateway.service.v1.common.OrderByExpression;
import org.hypertrace.gateway.service.v1.common.TimeAggregation;
import org.slf4j.Logger;
Expand Down Expand Up @@ -49,12 +51,17 @@ public class ExpressionContext {
private ImmutableMap<String, List<OrderByExpression>> sourceToMetricOrderByExpressionMap;
private ImmutableMap<String, Set<String>> sourceToMetricOrderByAttributeMap;

private ImmutableMap<String, List<OrderByExpression>> sourceToOrderByExpressionMap;

// filters
private final Filter filter;
private ImmutableMap<String, List<Expression>> sourceToFilterExpressionMap;
private ImmutableMap<String, Set<String>> sourceToFilterAttributeMap;
private ImmutableMap<String, Set<String>> filterAttributeToSourceMap;

// and filter
private boolean isAndFilter;

// group bys
private final List<Expression> groupBys;
private ImmutableMap<String, List<Expression>> sourceToGroupByExpressionMap;
Expand All @@ -64,6 +71,7 @@ public class ExpressionContext {
private final Map<String, Set<String>> allAttributesToSourcesMap = new HashMap<>();

public ExpressionContext(
GatewayServiceConfig gatewayServiceConfig,
Map<String, AttributeMetadata> attributeMetadataMap,
Filter filter,
List<Expression> selections,
Expand All @@ -82,6 +90,8 @@ public ExpressionContext(
buildSourceToFilterExpressionMaps();
buildSourceToOrderByExpressionMaps();
buildSourceToGroupByExpressionMaps();

this.isAndFilter = gatewayServiceConfig.isEntityAndFilterEnabled() && isAndFilter(filter);
}

public Map<String, List<Expression>> getSourceToSelectionExpressionMap() {
Expand Down Expand Up @@ -133,6 +143,10 @@ public Map<String, Set<String>> getSourceToMetricOrderByAttributeMap() {
return sourceToMetricOrderByAttributeMap;
}

public Map<String, List<OrderByExpression>> getSourceToOrderByExpressionMap() {
return sourceToOrderByExpressionMap;
}

public Map<String, List<Expression>> getSourceToFilterExpressionMap() {
return sourceToFilterExpressionMap;
}
Expand All @@ -145,6 +159,10 @@ public Map<String, Set<String>> getFilterAttributeToSourceMap() {
return filterAttributeToSourceMap;
}

public boolean isAndFilter() {
return isAndFilter;
}

public Map<String, Set<String>> getAllAttributesToSourcesMap() {
return allAttributesToSourcesMap;
}
Expand Down Expand Up @@ -173,6 +191,17 @@ private void buildSourceToFilterExpressionMaps() {
.build();
}

private boolean isAndFilter(Filter filter) {
Operator operator = filter.getOperator();
if (operator == Operator.AND) {
return filter.getChildFilterList().stream().allMatch(this::isAndFilter);
} else if (operator == Operator.OR) {
return false;
} else {
return true;
}
}

private void buildSourceToOrderByExpressionMaps() {
// Ensure that the OrderByExpression function alias matches that of a column in the selection or
// TimeAggregation, since the OrderByComparator uses the alias to match the column name in the
Expand Down Expand Up @@ -204,6 +233,8 @@ private void buildSourceToOrderByExpressionMaps() {
sourceToMetricOrderByAttributeMap =
buildSourceToAttributesMap(
convertOrderByExpressionToExpression(sourceToMetricOrderByExpressionMap));

sourceToOrderByExpressionMap = getDataSourceToOrderByExpressionMap(orderByExpressions);
}

private void buildSourceToGroupByExpressionMaps() {
Expand Down
39 changes: 39 additions & 0 deletions ...impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package org.hypertrace.gateway.service.common.config;

import com.typesafe.config.Config;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig;
import org.hypertrace.gateway.service.entity.config.LogConfig;

public class GatewayServiceConfig {
private static final String ENTITY_AND_FILTER_ENABLED_CONFIG_KEY = "filter.entity.and.enabled";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

entity.andFilter.enabled looks more readable.

private final EntityIdColumnsConfig entityIdColumnsConfig;
private final ScopeFilterConfigs scopeFilterConfigs;
private final LogConfig logConfig;
private final boolean isEntityAndFilterEnabled;

public GatewayServiceConfig(Config config) {
this.entityIdColumnsConfig = EntityIdColumnsConfig.fromConfig(config);
this.scopeFilterConfigs = new ScopeFilterConfigs(config);
this.logConfig = new LogConfig(config);
this.isEntityAndFilterEnabled =
config.hasPath(ENTITY_AND_FILTER_ENABLED_CONFIG_KEY)
? config.getBoolean(ENTITY_AND_FILTER_ENABLED_CONFIG_KEY)
: false;
}

public EntityIdColumnsConfig getEntityIdColumnsConfig() {
return entityIdColumnsConfig;
}

public ScopeFilterConfigs getScopeFilterConfigs() {
return scopeFilterConfigs;
}

public LogConfig getLogConfig() {
return logConfig;
}

public boolean isEntityAndFilterEnabled() {
return isEntityAndFilterEnabled;
}
}
10 changes: 5 additions & 5 deletions ...ava/org/hypertrace/gateway/service/common/datafetcher/EntityDataServiceEntityFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.hypertrace.gateway.service.common.util.ExpressionReader;
import org.hypertrace.gateway.service.entity.EntitiesRequestContext;
import org.hypertrace.gateway.service.entity.EntityKey;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig;
import org.hypertrace.gateway.service.v1.common.Value;
import org.hypertrace.gateway.service.v1.common.ValueType;
import org.hypertrace.gateway.service.v1.entity.EntitiesRequest;
Expand All @@ -39,15 +39,15 @@ public class EntityDataServiceEntityFetcher implements IEntityFetcher {

private final EntityQueryServiceClient entityQueryServiceClient;
private final AttributeMetadataProvider attributeMetadataProvider;
private final EntityIdColumnsConfigs entityIdColumnsConfigs;
private final EntityIdColumnsConfig entityIdColumnsConfig;

public EntityDataServiceEntityFetcher(
EntityQueryServiceClient entityQueryServiceClient,
AttributeMetadataProvider attributeMetadataProvider,
EntityIdColumnsConfigs entityIdColumnsConfigs) {
EntityIdColumnsConfig entityIdColumnsConfig) {
this.entityQueryServiceClient = entityQueryServiceClient;
this.attributeMetadataProvider = attributeMetadataProvider;
this.entityIdColumnsConfigs = entityIdColumnsConfigs;
this.entityIdColumnsConfig = entityIdColumnsConfig;
}

@Override
Expand All @@ -56,7 +56,7 @@ public EntityFetcherResponse getEntities(
List<String> entityIdAttributeIds =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());
Map<String, List<String>> requestedAliasesByEntityIdAttributeIds =
Expand Down
15 changes: 8 additions & 7 deletions ...ain/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import org.hypertrace.gateway.service.entity.EntitiesRequestContext;
import org.hypertrace.gateway.service.entity.EntitiesRequestValidator;
import org.hypertrace.gateway.service.entity.EntityKey;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig;
import org.hypertrace.gateway.service.v1.common.AggregatedMetricValue;
import org.hypertrace.gateway.service.v1.common.FunctionExpression;
import org.hypertrace.gateway.service.v1.common.FunctionType;
Expand All @@ -70,15 +70,15 @@ public class QueryServiceEntityFetcher implements IEntityFetcher {
private final EntitiesRequestValidator entitiesRequestValidator = new EntitiesRequestValidator();
private final QueryServiceClient queryServiceClient;
private final AttributeMetadataProvider attributeMetadataProvider;
private final EntityIdColumnsConfigs entityIdColumnsConfigs;
private final EntityIdColumnsConfig entityIdColumnsConfig;

public QueryServiceEntityFetcher(
QueryServiceClient queryServiceClient,
AttributeMetadataProvider attributeMetadataProvider,
EntityIdColumnsConfigs entityIdColumnsConfigs) {
EntityIdColumnsConfig entityIdColumnsConfig) {
this.queryServiceClient = queryServiceClient;
this.attributeMetadataProvider = attributeMetadataProvider;
this.entityIdColumnsConfigs = entityIdColumnsConfigs;
this.entityIdColumnsConfig = entityIdColumnsConfig;
}

@Override
Expand All @@ -95,7 +95,7 @@ public EntityFetcherResponse getEntities(
List<String> entityIdAttributeIds =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());
List<org.hypertrace.gateway.service.v1.common.Expression> aggregates =
Expand Down Expand Up @@ -179,6 +179,7 @@ public EntityFetcherResponse getEntities(
}
}
}

return new EntityFetcherResponse(entityBuilders);
}

Expand Down Expand Up @@ -352,7 +353,7 @@ public EntityFetcherResponse getTimeAggregatedMetrics(
List<String> idColumns =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());
String timeColumn =
Expand Down Expand Up @@ -525,7 +526,7 @@ public long getTotal(EntitiesRequestContext requestContext, EntitiesRequest enti
List<String> entityIdAttributeIds =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());

Expand Down
Loading
Loading