Skip to content
This repository has been archived by the owner on Jun 26, 2024. It is now read-only.

perf(entities): optimise AND filter query #193

Merged
merged 4 commits into from
Feb 28, 2024
Merged

perf(entities): optimise AND filter query #193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a700ec7
perf(entities): optimise AND filter query
skjindal93 Feb 21, 2024
151d11b
remove time tracking logs
skjindal93 Feb 21, 2024
bfa41b5
fix and filter for filter optimizing visitor
skjindal93 Feb 26, 2024
271eab2
fix tests
skjindal93 Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 7 additions & 14 deletions gateway-service-impl/src/main/java/org/hypertrace/gateway/service/GatewayServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,11 @@
import org.hypertrace.gateway.service.baseline.BaselineServiceQueryParser;
import org.hypertrace.gateway.service.common.AttributeMetadataProvider;
import org.hypertrace.gateway.service.common.RequestContext;
import org.hypertrace.gateway.service.common.config.ScopeFilterConfigs;
import org.hypertrace.gateway.service.common.config.GatewayServiceConfig;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceClient;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceV2Client;
import org.hypertrace.gateway.service.common.util.QueryServiceClient;
import org.hypertrace.gateway.service.entity.EntityService;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.entity.config.LogConfig;
import org.hypertrace.gateway.service.executor.QueryExecutorConfig;
import org.hypertrace.gateway.service.executor.QueryExecutorServiceFactory;
import org.hypertrace.gateway.service.explore.ExploreService;
Expand Down Expand Up @@ -91,7 +89,6 @@ public GatewayServiceImpl(Config appConfig, GrpcChannelRegistry grpcChannelRegis
new AttributeServiceClient(
grpcChannelRegistry.forPlaintextAddress(asConfig.getHost(), asConfig.getPort()));
AttributeMetadataProvider attributeMetadataProvider = new AttributeMetadataProvider(asClient);
EntityIdColumnsConfigs entityIdColumnsConfigs = EntityIdColumnsConfigs.fromConfig(appConfig);

Config untypedQsConfig = appConfig.getConfig(QUERY_SERVICE_CONFIG_KEY);
QueryServiceConfig qsConfig = new QueryServiceConfig(untypedQsConfig);
Expand Down Expand Up @@ -124,41 +121,37 @@ public GatewayServiceImpl(Config appConfig, GrpcChannelRegistry grpcChannelRegis
EntityTypesProvider entityTypesProvider =
new EntityTypesProvider(entityTypeServiceClient, entityTypeServiceV2Client);

ScopeFilterConfigs scopeFilterConfigs = new ScopeFilterConfigs(appConfig);
LogConfig logConfig = new LogConfig(appConfig);
GatewayServiceConfig gatewayServiceConfig = new GatewayServiceConfig(appConfig);
this.traceService =
new TracesService(
queryServiceClient, attributeMetadataProvider, scopeFilterConfigs, queryExecutor);
gatewayServiceConfig, queryServiceClient, attributeMetadataProvider, queryExecutor);
this.spanService =
new SpanService(queryServiceClient, attributeMetadataProvider, queryExecutor);
this.entityService =
new EntityService(
gatewayServiceConfig,
queryServiceClient,
eqsClient,
eqsStub,
attributeMetadataProvider,
entityIdColumnsConfigs,
scopeFilterConfigs,
logConfig,
queryExecutor);
this.exploreService =
new ExploreService(
gatewayServiceConfig,
queryServiceClient,
eqsClient,
attributeMetadataProvider,
scopeFilterConfigs,
entityIdColumnsConfigs,
entityTypesProvider);
BaselineServiceQueryParser baselineServiceQueryParser =
new BaselineServiceQueryParser(attributeMetadataProvider);
BaselineServiceQueryExecutor baselineServiceQueryExecutor =
new BaselineServiceQueryExecutor(queryServiceClient);
this.baselineService =
new BaselineServiceImpl(
gatewayServiceConfig,
attributeMetadataProvider,
baselineServiceQueryParser,
baselineServiceQueryExecutor,
entityIdColumnsConfigs);
baselineServiceQueryExecutor);
this.logEventsService = new LogEventsService(queryServiceClient, attributeMetadataProvider);
initMetrics();
}
Expand Down
18 changes: 9 additions & 9 deletions ...rvice-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
import org.hypertrace.gateway.service.baseline.lib.BaselineCalculator;
import org.hypertrace.gateway.service.common.AttributeMetadataProvider;
import org.hypertrace.gateway.service.common.RequestContext;
import org.hypertrace.gateway.service.common.config.GatewayServiceConfig;
import org.hypertrace.gateway.service.common.util.AttributeMetadataUtil;
import org.hypertrace.gateway.service.common.util.QueryExpressionUtil;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.v1.baseline.Baseline;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse;
Expand All @@ -40,25 +40,25 @@
*/
public class BaselineServiceImpl implements BaselineService {

private static final BaselineEntitiesRequestValidator baselineEntitiesRequestValidator =
private static final BaselineEntitiesRequestValidator BASELINE_ENTITIES_REQUEST_VALIDATOR =
new BaselineEntitiesRequestValidator();

protected static final long DAY_IN_MILLIS = 86400000L;

private final GatewayServiceConfig gatewayServiceConfig;
private final AttributeMetadataProvider attributeMetadataProvider;
private final BaselineServiceQueryParser baselineServiceQueryParser;
private final BaselineServiceQueryExecutor baselineServiceQueryExecutor;
private final EntityIdColumnsConfigs entityIdColumnsConfigs;

public BaselineServiceImpl(
GatewayServiceConfig gatewayServiceConfig,
AttributeMetadataProvider attributeMetadataProvider,
BaselineServiceQueryParser baselineServiceQueryParser,
BaselineServiceQueryExecutor baselineServiceQueryExecutor,
EntityIdColumnsConfigs entityIdColumnsConfigs) {
BaselineServiceQueryExecutor baselineServiceQueryExecutor) {
this.gatewayServiceConfig = gatewayServiceConfig;
this.attributeMetadataProvider = attributeMetadataProvider;
this.baselineServiceQueryParser = baselineServiceQueryParser;
this.baselineServiceQueryExecutor = baselineServiceQueryExecutor;
this.entityIdColumnsConfigs = entityIdColumnsConfigs;
}

public BaselineEntitiesResponse getBaselineForEntities(
Expand All @@ -67,7 +67,7 @@ public BaselineEntitiesResponse getBaselineForEntities(
Map<String, AttributeMetadata> attributeMetadataMap =
attributeMetadataProvider.getAttributesMetadata(
requestContext, originalRequest.getEntityType());
baselineEntitiesRequestValidator.validate(originalRequest, attributeMetadataMap);
BASELINE_ENTITIES_REQUEST_VALIDATOR.validate(originalRequest, attributeMetadataMap);
String timeColumn =
AttributeMetadataUtil.getTimestampAttributeId(
attributeMetadataProvider, requestContext, originalRequest.getEntityType());
Expand All @@ -91,7 +91,7 @@ public BaselineEntitiesResponse getBaselineForEntities(
List<String> entityIdAttributes =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
gatewayServiceConfig.getEntityIdColumnsConfig(),
requestContext,
originalRequest.getEntityType());
QueryRequest aggQueryRequest =
Expand Down Expand Up @@ -134,7 +134,7 @@ public BaselineEntitiesResponse getBaselineForEntities(
List<String> entityIdAttributes =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
gatewayServiceConfig.getEntityIdColumnsConfig(),
requestContext,
originalRequest.getEntityType());
QueryRequest timeSeriesQueryRequest =
Expand Down
31 changes: 31 additions & 0 deletions ...y-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
import java.util.stream.Stream;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

logic class

import org.hypertrace.core.attribute.service.v1.AttributeMetadata;
import org.hypertrace.core.attribute.service.v1.AttributeSource;
import org.hypertrace.gateway.service.common.config.GatewayServiceConfig;
import org.hypertrace.gateway.service.common.util.ExpressionReader;
import org.hypertrace.gateway.service.common.util.OrderByUtil;
import org.hypertrace.gateway.service.v1.common.Expression;
import org.hypertrace.gateway.service.v1.common.Filter;
import org.hypertrace.gateway.service.v1.common.Operator;
import org.hypertrace.gateway.service.v1.common.OrderByExpression;
import org.hypertrace.gateway.service.v1.common.TimeAggregation;
import org.slf4j.Logger;
Expand Down Expand Up @@ -49,12 +51,17 @@
private ImmutableMap<String, List<OrderByExpression>> sourceToMetricOrderByExpressionMap;
private ImmutableMap<String, Set<String>> sourceToMetricOrderByAttributeMap;

private ImmutableMap<String, List<OrderByExpression>> sourceToOrderByExpressionMap;

// filters
private final Filter filter;
private ImmutableMap<String, List<Expression>> sourceToFilterExpressionMap;
private ImmutableMap<String, Set<String>> sourceToFilterAttributeMap;
private ImmutableMap<String, Set<String>> filterAttributeToSourceMap;

// and filter
private boolean isAndFilter;

// group bys
private final List<Expression> groupBys;
private ImmutableMap<String, List<Expression>> sourceToGroupByExpressionMap;
Expand All @@ -64,6 +71,7 @@
private final Map<String, Set<String>> allAttributesToSourcesMap = new HashMap<>();

public ExpressionContext(
GatewayServiceConfig gatewayServiceConfig,
Map<String, AttributeMetadata> attributeMetadataMap,
Filter filter,
List<Expression> selections,
Expand All @@ -82,6 +90,8 @@
buildSourceToFilterExpressionMaps();
buildSourceToOrderByExpressionMaps();
buildSourceToGroupByExpressionMaps();

this.isAndFilter = gatewayServiceConfig.isEntityAndFilterEnabled() && isAndFilter(filter);
}

public Map<String, List<Expression>> getSourceToSelectionExpressionMap() {
Expand Down Expand Up @@ -133,6 +143,10 @@
return sourceToMetricOrderByAttributeMap;
}

public Map<String, List<OrderByExpression>> getSourceToOrderByExpressionMap() {
return sourceToOrderByExpressionMap;

Check warning on line 147 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java#L147 

Added line #L147 was not covered by tests
}

public Map<String, List<Expression>> getSourceToFilterExpressionMap() {
return sourceToFilterExpressionMap;
}
Expand All @@ -145,6 +159,10 @@
return filterAttributeToSourceMap;
}

public boolean isAndFilter() {
return isAndFilter;
}

public Map<String, Set<String>> getAllAttributesToSourcesMap() {
return allAttributesToSourcesMap;
}
Expand Down Expand Up @@ -173,6 +191,17 @@
.build();
}

private boolean isAndFilter(Filter filter) {
Operator operator = filter.getOperator();

Check warning on line 195 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java#L195 

Added line #L195 was not covered by tests
if (operator == Operator.AND) {
return filter.getChildFilterList().stream().allMatch(this::isAndFilter);

Check warning on line 197 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java#L197 

Added line #L197 was not covered by tests
} else if (operator == Operator.OR) {
return false;

Check warning on line 199 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java#L199 

Added line #L199 was not covered by tests
} else {
return true;

Check warning on line 201 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/ExpressionContext.java#L201 

Added line #L201 was not covered by tests
}
}

private void buildSourceToOrderByExpressionMaps() {
// Ensure that the OrderByExpression function alias matches that of a column in the selection or
// TimeAggregation, since the OrderByComparator uses the alias to match the column name in the
Expand Down Expand Up @@ -204,6 +233,8 @@
sourceToMetricOrderByAttributeMap =
buildSourceToAttributesMap(
convertOrderByExpressionToExpression(sourceToMetricOrderByExpressionMap));

sourceToOrderByExpressionMap = getDataSourceToOrderByExpressionMap(orderByExpressions);
}

private void buildSourceToGroupByExpressionMaps() {
Expand Down
39 changes: 39 additions & 0 deletions ...impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package org.hypertrace.gateway.service.common.config;

import com.typesafe.config.Config;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig;
import org.hypertrace.gateway.service.entity.config.LogConfig;

public class GatewayServiceConfig {
private static final String ENTITY_AND_FILTER_ENABLED_CONFIG_KEY = "filter.entity.and.enabled";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

entity.andFilter.enabled looks more readable.

private final EntityIdColumnsConfig entityIdColumnsConfig;
private final ScopeFilterConfigs scopeFilterConfigs;
private final LogConfig logConfig;
private final boolean isEntityAndFilterEnabled;

public GatewayServiceConfig(Config config) {
this.entityIdColumnsConfig = EntityIdColumnsConfig.fromConfig(config);
this.scopeFilterConfigs = new ScopeFilterConfigs(config);
this.logConfig = new LogConfig(config);
this.isEntityAndFilterEnabled =
config.hasPath(ENTITY_AND_FILTER_ENABLED_CONFIG_KEY)
? config.getBoolean(ENTITY_AND_FILTER_ENABLED_CONFIG_KEY)

Check warning on line 20 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java#L20 

Added line #L20 was not covered by tests
: false;
}

public EntityIdColumnsConfig getEntityIdColumnsConfig() {
return entityIdColumnsConfig;
}

public ScopeFilterConfigs getScopeFilterConfigs() {
return scopeFilterConfigs;
}

public LogConfig getLogConfig() {
return logConfig;

Check warning on line 33 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java#L33 

Added line #L33 was not covered by tests
}

public boolean isEntityAndFilterEnabled() {
return isEntityAndFilterEnabled;

Check warning on line 37 in gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java

View check run for this annotation

Codecov / codecov/patch

gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/config/GatewayServiceConfig.java#L37 

Added line #L37 was not covered by tests
}
}
10 changes: 5 additions & 5 deletions ...ava/org/hypertrace/gateway/service/common/datafetcher/EntityDataServiceEntityFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.hypertrace.gateway.service.common.util.ExpressionReader;
import org.hypertrace.gateway.service.entity.EntitiesRequestContext;
import org.hypertrace.gateway.service.entity.EntityKey;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig;
import org.hypertrace.gateway.service.v1.common.Value;
import org.hypertrace.gateway.service.v1.common.ValueType;
import org.hypertrace.gateway.service.v1.entity.EntitiesRequest;
Expand All @@ -39,15 +39,15 @@ public class EntityDataServiceEntityFetcher implements IEntityFetcher {

private final EntityQueryServiceClient entityQueryServiceClient;
private final AttributeMetadataProvider attributeMetadataProvider;
private final EntityIdColumnsConfigs entityIdColumnsConfigs;
private final EntityIdColumnsConfig entityIdColumnsConfig;

public EntityDataServiceEntityFetcher(
EntityQueryServiceClient entityQueryServiceClient,
AttributeMetadataProvider attributeMetadataProvider,
EntityIdColumnsConfigs entityIdColumnsConfigs) {
EntityIdColumnsConfig entityIdColumnsConfig) {
this.entityQueryServiceClient = entityQueryServiceClient;
this.attributeMetadataProvider = attributeMetadataProvider;
this.entityIdColumnsConfigs = entityIdColumnsConfigs;
this.entityIdColumnsConfig = entityIdColumnsConfig;
}

@Override
Expand All @@ -56,7 +56,7 @@ public EntityFetcherResponse getEntities(
List<String> entityIdAttributeIds =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());
Map<String, List<String>> requestedAliasesByEntityIdAttributeIds =
Expand Down
15 changes: 8 additions & 7 deletions ...ain/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import org.hypertrace.gateway.service.entity.EntitiesRequestContext;
import org.hypertrace.gateway.service.entity.EntitiesRequestValidator;
import org.hypertrace.gateway.service.entity.EntityKey;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfig;
import org.hypertrace.gateway.service.v1.common.AggregatedMetricValue;
import org.hypertrace.gateway.service.v1.common.FunctionExpression;
import org.hypertrace.gateway.service.v1.common.FunctionType;
Expand All @@ -70,15 +70,15 @@ public class QueryServiceEntityFetcher implements IEntityFetcher {
private final EntitiesRequestValidator entitiesRequestValidator = new EntitiesRequestValidator();
private final QueryServiceClient queryServiceClient;
private final AttributeMetadataProvider attributeMetadataProvider;
private final EntityIdColumnsConfigs entityIdColumnsConfigs;
private final EntityIdColumnsConfig entityIdColumnsConfig;

public QueryServiceEntityFetcher(
QueryServiceClient queryServiceClient,
AttributeMetadataProvider attributeMetadataProvider,
EntityIdColumnsConfigs entityIdColumnsConfigs) {
EntityIdColumnsConfig entityIdColumnsConfig) {
this.queryServiceClient = queryServiceClient;
this.attributeMetadataProvider = attributeMetadataProvider;
this.entityIdColumnsConfigs = entityIdColumnsConfigs;
this.entityIdColumnsConfig = entityIdColumnsConfig;
}

@Override
Expand All @@ -95,7 +95,7 @@ public EntityFetcherResponse getEntities(
List<String> entityIdAttributeIds =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());
List<org.hypertrace.gateway.service.v1.common.Expression> aggregates =
Expand Down Expand Up @@ -179,6 +179,7 @@ public EntityFetcherResponse getEntities(
}
}
}

return new EntityFetcherResponse(entityBuilders);
}

Expand Down Expand Up @@ -352,7 +353,7 @@ public EntityFetcherResponse getTimeAggregatedMetrics(
List<String> idColumns =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());
String timeColumn =
Expand Down Expand Up @@ -525,7 +526,7 @@ public long getTotal(EntitiesRequestContext requestContext, EntitiesRequest enti
List<String> entityIdAttributeIds =
AttributeMetadataUtil.getIdAttributeIds(
attributeMetadataProvider,
entityIdColumnsConfigs,
entityIdColumnsConfig,
requestContext,
entitiesRequest.getEntityType());

Expand Down
Loading
Loading