From b61884d3c69d09c40e109881c34749ff7ef7be7d Mon Sep 17 00:00:00 2001 From: pavan-traceable <73101820+pavan-traceable@users.noreply.github.com> Date: Wed, 6 Jan 2021 18:01:11 +0530 Subject: [PATCH] Add baseline api and implementaion for entities (#55) * Add baseline api and implementation for entities --- .../gateway/service/v1/baseline.proto | 50 +++ .../gateway/service/v1/entities.proto | 1 + .../gateway/service/v1/gateway_service.proto | 3 + gateway-service-impl/build.gradle.kts | 1 + .../gateway/service/GatewayServiceImpl.java | 43 +- .../service/baseline/BaselineCalculator.java | 57 +++ .../BaselineEntitiesRequestValidator.java | 40 ++ .../baseline/BaselineRequestContext.java | 24 ++ .../service/baseline/BaselineService.java | 11 + .../service/baseline/BaselineServiceImpl.java | 369 ++++++++++++++++++ .../BaselineServiceQueryExecutor.java | 24 ++ .../baseline/BaselineServiceQueryParser.java | 212 ++++++++++ .../common/converters/QueryRequestUtil.java | 4 +- .../EntityDataServiceEntityFetcher.java | 1 + .../QueryServiceEntityFetcher.java | 53 +-- .../util/MetricAggregationFunctionUtil.java | 28 ++ .../gateway/service/entity/EntityService.java | 4 +- .../service/explore/RequestHandler.java | 29 +- .../baseline/BaselineCalculatorTest.java | 48 +++ .../BaselineEntitiesRequestValidatorTest.java | 71 ++++ .../baseline/BaselineServiceImplTest.java | 195 +++++++++ .../BaselineServiceQueryParserTest.java | 131 +++++++ .../QueryServiceEntityFetcherTests.java | 5 + 23 files changed, 1344 insertions(+), 60 deletions(-) create mode 100644 gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/baseline.proto create mode 100644 gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineCalculator.java create mode 100644 gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidator.java create mode 100644 gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineRequestContext.java create mode 100644 gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineService.java create mode 100644 gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceImpl.java create mode 100644 gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryExecutor.java create mode 100644 gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParser.java create mode 100644 gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineCalculatorTest.java create mode 100644 gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidatorTest.java create mode 100644 gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceImplTest.java create mode 100644 gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParserTest.java diff --git a/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/baseline.proto b/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/baseline.proto new file mode 100644 index 00000000..e7b0f964 --- /dev/null +++ b/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/baseline.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.hypertrace.gateway.service.v1.baseline"; + +package org.hypertrace.gateway.service.v1.baseline; + +import "org/hypertrace/gateway/service/v1/gateway_query.proto"; + +message BaselineEntitiesRequest { + string entity_type = 1; + sfixed64 start_time_millis = 2; + sfixed64 end_time_millis = 3; + + repeated string entity_ids = 4; + repeated org.hypertrace.gateway.service.v1.common.FunctionExpression baseline_aggregate_request = 7; + repeated BaselineTimeAggregation baseline_metric_series_request = 8; +} + +message BaselineEntitiesResponse { + repeated BaselineEntity baseline_entity = 1; +} + +message BaselineEntity { + string id = 1; + string entity_type = 2; + map baseline_aggregate_metric = 5; + map baseline_metric_series = 6; +} + +message BaselineTimeAggregation { + org.hypertrace.gateway.service.v1.common.Period period = 1; + org.hypertrace.gateway.service.v1.common.FunctionExpression aggregation = 2; +} + +message Baseline { + org.hypertrace.gateway.service.v1.common.Value value = 1; + org.hypertrace.gateway.service.v1.common.Value lower_bound = 2; + org.hypertrace.gateway.service.v1.common.Value upper_bound = 3; +} + +message BaselineMetricSeries { + repeated BaselineInterval baseline_value = 1; +} + +message BaselineInterval { + sfixed64 start_time_millis = 1; + sfixed64 end_time_millis = 2; + Baseline baseline = 3; +} \ No newline at end of file diff --git a/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/entities.proto b/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/entities.proto index d8e79c28..e1a5dafb 100644 --- a/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/entities.proto +++ b/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/entities.proto @@ -10,6 +10,7 @@ import "org/hypertrace/gateway/service/v1/gateway_query.proto"; // Entity + metric data message Entity { + string id = 1; string entity_type = 2; map attribute = 4; map metric = 5; diff --git a/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/gateway_service.proto b/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/gateway_service.proto index 4ec0adb7..e31300a3 100644 --- a/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/gateway_service.proto +++ b/gateway-service-api/src/main/proto/org/hypertrace/gateway/service/v1/gateway_service.proto @@ -9,12 +9,15 @@ import "org/hypertrace/gateway/service/v1/entities.proto"; import "org/hypertrace/gateway/service/v1/explore.proto"; import "org/hypertrace/gateway/service/v1/spans.proto"; import "org/hypertrace/gateway/service/v1/traces.proto"; +import "org/hypertrace/gateway/service/v1/baseline.proto"; service GatewayService { rpc getEntities (org.hypertrace.gateway.service.v1.entity.EntitiesRequest) returns (org.hypertrace.gateway.service.v1.entity.EntitiesResponse) {} rpc updateEntity (org.hypertrace.gateway.service.v1.entity.UpdateEntityRequest) returns (org.hypertrace.gateway.service.v1.entity.UpdateEntityResponse) {} + rpc getBaselineForEntities(org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest) + returns (org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse) {} rpc explore (org.hypertrace.gateway.service.v1.explore.ExploreRequest) returns (org.hypertrace.gateway.service.v1.explore.ExploreResponse) {} rpc getTraces (org.hypertrace.gateway.service.v1.trace.TracesRequest) diff --git a/gateway-service-impl/build.gradle.kts b/gateway-service-impl/build.gradle.kts index 3625faee..fbc58f21 100644 --- a/gateway-service-impl/build.gradle.kts +++ b/gateway-service-impl/build.gradle.kts @@ -35,6 +35,7 @@ dependencies { // Common utilities implementation("org.apache.commons:commons-lang3:3.10") + implementation("org.apache.commons:commons-math:2.2") implementation("com.google.protobuf:protobuf-java-util:3.13.0") // Metrics diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/GatewayServiceImpl.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/GatewayServiceImpl.java index 73e5c127..b138461a 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/GatewayServiceImpl.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/GatewayServiceImpl.java @@ -14,6 +14,8 @@ import org.hypertrace.core.query.service.client.QueryServiceConfig; import org.hypertrace.entity.query.service.client.EntityQueryServiceClient; import org.hypertrace.entity.service.client.config.EntityServiceClientConfig; +import org.hypertrace.gateway.service.baseline.BaselineServiceQueryExecutor; +import org.hypertrace.gateway.service.baseline.BaselineServiceQueryParser; import org.hypertrace.gateway.service.common.AttributeMetadataProvider; import org.hypertrace.gateway.service.common.RequestContext; import org.hypertrace.gateway.service.common.config.ScopeFilterConfigs; @@ -21,6 +23,8 @@ import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs; import org.hypertrace.gateway.service.entity.config.LogConfig; import org.hypertrace.gateway.service.explore.ExploreService; +import org.hypertrace.gateway.service.baseline.BaselineService; +import org.hypertrace.gateway.service.baseline.BaselineServiceImpl; import org.hypertrace.gateway.service.span.SpanService; import org.hypertrace.gateway.service.trace.TracesService; import org.hypertrace.gateway.service.v1.entity.EntitiesResponse; @@ -28,6 +32,8 @@ import org.hypertrace.gateway.service.v1.entity.UpdateEntityResponse; import org.hypertrace.gateway.service.v1.explore.ExploreRequest; import org.hypertrace.gateway.service.v1.explore.ExploreResponse; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse; import org.hypertrace.gateway.service.v1.span.SpansResponse; import org.hypertrace.gateway.service.v1.trace.TracesResponse; import org.slf4j.Logger; @@ -51,6 +57,7 @@ public class GatewayServiceImpl extends GatewayServiceGrpc.GatewayServiceImplBas private final SpanService spanService; private final EntityService entityService; private final ExploreService exploreService; + private final BaselineService baselineService; public GatewayServiceImpl(Config appConfig) { AttributeServiceClientConfig asConfig = AttributeServiceClientConfig.from(appConfig); @@ -83,6 +90,9 @@ public GatewayServiceImpl(Config appConfig) { this.exploreService = new ExploreService(queryServiceClient, qsRequestTimeout, attributeMetadataProvider, scopeFilterConfigs); + BaselineServiceQueryParser baselineServiceQueryParser = new BaselineServiceQueryParser(attributeMetadataProvider); + BaselineServiceQueryExecutor baselineServiceQueryExecutor = new BaselineServiceQueryExecutor(qsRequestTimeout, queryServiceClient); + this.baselineService = new BaselineServiceImpl(attributeMetadataProvider, baselineServiceQueryParser, baselineServiceQueryExecutor, entityIdColumnsConfigs); } private static int getRequestTimeoutMillis(Config config) { @@ -184,9 +194,7 @@ public void getEntities( org.hypertrace.core.grpcutils.context.RequestContext.CURRENT.get() .getRequestHeaders()); - if (LOG.isDebugEnabled()) { - LOG.debug("Received response: {}", response); - } + LOG.debug("Received response: {}", response); responseObserver.onNext(response); responseObserver.onCompleted(); @@ -229,6 +237,35 @@ public void updateEntity( } } + @Override + public void getBaselineForEntities( + BaselineEntitiesRequest request, StreamObserver responseObserver) { + Optional tenantId = + org.hypertrace.core.grpcutils.context.RequestContext.CURRENT.get().getTenantId(); + if (tenantId.isEmpty()) { + responseObserver.onError(new ServiceException("Tenant id is missing in the request.")); + return; + } + + try { + BaselineEntitiesResponse response = + baselineService.getBaselineForEntities( + tenantId.get(), + request, + org.hypertrace.core.grpcutils.context.RequestContext.CURRENT + .get() + .getRequestHeaders()); + + LOG.debug("Received response: {}", response); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (Exception e) { + LOG.error("Error while handling entities request: {}.", request, e); + responseObserver.onError(e); + } + } + @Override public void explore(ExploreRequest request, StreamObserver responseObserver) { Optional tenantId = diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineCalculator.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineCalculator.java new file mode 100644 index 00000000..1cf15221 --- /dev/null +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineCalculator.java @@ -0,0 +1,57 @@ +package org.hypertrace.gateway.service.baseline; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.commons.math.stat.descriptive.moment.StandardDeviation; +import org.apache.commons.math.stat.descriptive.rank.Median; +import org.hypertrace.gateway.service.v1.baseline.Baseline; +import org.hypertrace.gateway.service.v1.common.Value; +import org.hypertrace.gateway.service.v1.common.ValueType; + +import java.util.List; + +public class BaselineCalculator { + + public static Baseline getBaseline(List metricValues) { + if (metricValues.isEmpty()) { + return Baseline.getDefaultInstance(); + } + double[] values = getValuesInDouble(metricValues); + return getBaseline(values); + } + + private static Baseline getBaseline(double[] metricValueArray) { + Median median = new Median(); + StandardDeviation standardDeviation = new StandardDeviation(); + double medianValue = median.evaluate(metricValueArray); + double sd = standardDeviation.evaluate(metricValueArray); + double lowerBound = medianValue - (2 * sd); + if (lowerBound < 0) { + lowerBound = 0; + } + double upperBound = medianValue + (2 * sd); + Baseline baseline = + Baseline.newBuilder() + .setLowerBound( + Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(lowerBound).build()) + .setUpperBound( + Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(upperBound).build()) + .setValue( + Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(medianValue).build()) + .build(); + return baseline; + } + + @VisibleForTesting + private static double[] getValuesInDouble(List metricValues) { + ValueType valueType = metricValues.get(0).getValueType(); + switch (valueType) { + case LONG: + return metricValues.stream().mapToDouble(value -> (double) value.getLong()).toArray(); + case DOUBLE: + return metricValues.stream().mapToDouble(value -> value.getDouble()).toArray(); + default: + throw new IllegalArgumentException("Unsupported valueType " + valueType); + } + } +} diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidator.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidator.java new file mode 100644 index 00000000..1a7527c5 --- /dev/null +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidator.java @@ -0,0 +1,40 @@ +package org.hypertrace.gateway.service.baseline; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; +import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.gateway.service.common.validators.request.RequestValidator; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class BaselineEntitiesRequestValidator extends RequestValidator { + private static final Logger LOG = LoggerFactory.getLogger(BaselineEntitiesRequestValidator.class); + + @Override + public void validate( + BaselineEntitiesRequest request, Map attributeMetadataMap) { + Preconditions.checkArgument( + StringUtils.isNotBlank(request.getEntityType()), "EntityType is mandatory in the request."); + + Preconditions.checkArgument(request.getEntityIdsCount() > 0, "EntityIds cannot be empty"); + + Preconditions.checkArgument( + (request.getBaselineAggregateRequestCount() > 0 + || request.getBaselineMetricSeriesRequestCount() > 0), + "Both Selection list and TimeSeries list can't be empty in the request."); + + Preconditions.checkArgument( + request.getStartTimeMillis() > 0 + && request.getEndTimeMillis() > 0 + && request.getStartTimeMillis() < request.getEndTimeMillis(), + "Invalid time range. Both start and end times have to be valid timestamps."); + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineRequestContext.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineRequestContext.java new file mode 100644 index 00000000..37b9ea23 --- /dev/null +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineRequestContext.java @@ -0,0 +1,24 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.gateway.service.common.RequestContext; +import org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation; + +import java.util.HashMap; +import java.util.Map; + +public class BaselineRequestContext extends RequestContext { + + private final Map aliasToTimeAggregation = new HashMap(); + + public BaselineRequestContext(String tenantId, Map headers) { + super(tenantId, headers); + } + + public void mapAliasToTimeAggregation(String alias, BaselineTimeAggregation timeAggregation) { + aliasToTimeAggregation.put(alias, timeAggregation); + } + + public BaselineTimeAggregation getTimeAggregationByAlias(String alias) { + return aliasToTimeAggregation.get(alias); + } +} diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineService.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineService.java new file mode 100644 index 00000000..3dd54e96 --- /dev/null +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineService.java @@ -0,0 +1,11 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse; + +import java.util.Map; + +public interface BaselineService { + BaselineEntitiesResponse getBaselineForEntities( + String tenantId, BaselineEntitiesRequest originalRequest, Map requestHeaders); +} diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceImpl.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceImpl.java new file mode 100644 index 00000000..7ace330b --- /dev/null +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceImpl.java @@ -0,0 +1,369 @@ +package org.hypertrace.gateway.service.baseline; + +import com.google.common.annotations.VisibleForTesting; +import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.ResultSetChunk; +import org.hypertrace.gateway.service.common.AttributeMetadataProvider; +import org.hypertrace.gateway.service.common.util.AttributeMetadataUtil; +import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs; +import org.hypertrace.gateway.service.v1.baseline.Baseline; +import org.hypertrace.gateway.service.v1.baseline.BaselineInterval; +import org.hypertrace.gateway.service.v1.baseline.BaselineMetricSeries; +import org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation; +import org.hypertrace.gateway.service.v1.common.Expression; +import org.hypertrace.gateway.service.v1.common.FunctionExpression; +import org.hypertrace.gateway.service.v1.common.Period; +import org.hypertrace.gateway.service.v1.common.TimeAggregation; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntity; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse; +import org.hypertrace.gateway.service.v1.common.Value; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * This service calculates baseline values for an Entity across time range. This converts both + * aggregate functions and time series functions in selection expression to time series data and + * generate baseline, lower bound and upper bound. It also changes Time series requests to get more + * data points for proper baseline calculation + */ +public class BaselineServiceImpl implements BaselineService { + + private static final BaselineEntitiesRequestValidator baselineEntitiesRequestValidator = + new BaselineEntitiesRequestValidator(); + + protected static final long DAY_IN_MILLIS = 86400000L; + + private final AttributeMetadataProvider attributeMetadataProvider; + private final BaselineServiceQueryParser baselineServiceQueryParser; + private final BaselineServiceQueryExecutor baselineServiceQueryExecutor; + private final EntityIdColumnsConfigs entityIdColumnsConfigs; + + public BaselineServiceImpl( + AttributeMetadataProvider attributeMetadataProvider, + BaselineServiceQueryParser baselineServiceQueryParser, + BaselineServiceQueryExecutor baselineServiceQueryExecutor, + EntityIdColumnsConfigs entityIdColumnsConfigs) { + this.attributeMetadataProvider = attributeMetadataProvider; + this.baselineServiceQueryParser = baselineServiceQueryParser; + this.baselineServiceQueryExecutor = baselineServiceQueryExecutor; + this.entityIdColumnsConfigs = entityIdColumnsConfigs; + } + + public BaselineEntitiesResponse getBaselineForEntities( + String tenantId, + BaselineEntitiesRequest originalRequest, + Map requestHeaders) { + BaselineRequestContext requestContext = + getRequestContext(tenantId, requestHeaders, originalRequest); + Map attributeMetadataMap = + attributeMetadataProvider.getAttributesMetadata( + requestContext, originalRequest.getEntityType()); + baselineEntitiesRequestValidator.validate(originalRequest, attributeMetadataMap); + String timeColumn = + AttributeMetadataUtil.getTimestampAttributeId( + attributeMetadataProvider, requestContext, originalRequest.getEntityType()); + Map baselineEntityAggregatedMetricsMap = new HashMap<>(); + Map baselineEntityTimeSeriesMap = new HashMap<>(); + + if (originalRequest.getBaselineAggregateRequestCount() > 0) { + // Aggregated Functions data + Period aggTimePeriod = + getPeriod(originalRequest.getStartTimeMillis(), originalRequest.getEndTimeMillis()); + long periodSecs = getPeriodInSecs(aggTimePeriod); + List timeAggregations = getTimeAggregationsForAggregateExpr(originalRequest); + updateAliasMap(requestContext, timeAggregations); + // Take more data to calculate baseline and standard deviation. + long seriesStartTime = + getUpdatedStartTime( + originalRequest.getStartTimeMillis(), originalRequest.getEndTimeMillis()); + long seriesEndTime = originalRequest.getStartTimeMillis(); + List entityIdAttributes = + AttributeMetadataUtil.getIdAttributeIds( + attributeMetadataProvider, + entityIdColumnsConfigs, + requestContext, + originalRequest.getEntityType()); + QueryRequest aggQueryRequest = + baselineServiceQueryParser.getQueryRequest( + seriesStartTime, + seriesEndTime, + originalRequest.getEntityIdsList(), + timeColumn, + timeAggregations, + periodSecs, + entityIdAttributes); + Iterator aggResponseChunkIterator = + baselineServiceQueryExecutor.executeQuery(requestHeaders, aggQueryRequest); + BaselineEntitiesResponse aggEntitiesResponse = + baselineServiceQueryParser.parseQueryResponse( + aggResponseChunkIterator, + requestContext, + entityIdAttributes.size(), + originalRequest.getEntityType(), + originalRequest.getStartTimeMillis(), + originalRequest.getEndTimeMillis()); + baselineEntityAggregatedMetricsMap = getEntitiesMapFromAggResponse(aggEntitiesResponse); + } + + // Time Series data + if (originalRequest.getBaselineMetricSeriesRequestCount() > 0) { + Period timeSeriesPeriod = + getTimeSeriesPeriod(originalRequest.getBaselineMetricSeriesRequestList()); + long periodSecs = getPeriodInSecs(timeSeriesPeriod); + List timeAggregations = + getTimeAggregationsForTimeSeriesExpr(originalRequest); + long seriesStartTime = + getUpdatedStartTime( + originalRequest.getStartTimeMillis(), originalRequest.getEndTimeMillis()); + long seriesEndTime = originalRequest.getStartTimeMillis(); + List entityIdAttributes = + AttributeMetadataUtil.getIdAttributeIds( + attributeMetadataProvider, + entityIdColumnsConfigs, + requestContext, + originalRequest.getEntityType()); + QueryRequest timeSeriesQueryRequest = + baselineServiceQueryParser.getQueryRequest( + seriesStartTime, + seriesEndTime, + originalRequest.getEntityIdsList(), + timeColumn, + timeAggregations, + periodSecs, + entityIdAttributes); + Iterator timeSeriesChunkIterator = + baselineServiceQueryExecutor.executeQuery(requestHeaders, timeSeriesQueryRequest); + BaselineEntitiesResponse timeSeriesEntitiesResponse = + baselineServiceQueryParser.parseQueryResponse( + timeSeriesChunkIterator, + requestContext, + entityIdAttributes.size(), + originalRequest.getEntityType(), + originalRequest.getStartTimeMillis(), + originalRequest.getEndTimeMillis()); + baselineEntityTimeSeriesMap = + getEntitiesMapFromTimeSeriesResponse( + timeSeriesEntitiesResponse, + originalRequest.getStartTimeMillis(), + originalRequest.getEndTimeMillis(), + periodSecs); + } + + return mergeEntities(baselineEntityAggregatedMetricsMap, baselineEntityTimeSeriesMap); + } + + private void updateAliasMap( + BaselineRequestContext requestContext, List timeAggregations) { + timeAggregations.forEach( + (timeAggregation -> + requestContext.mapAliasToTimeAggregation( + timeAggregation.getAggregation().getFunction().getAlias(), + getBaselineAggregation(timeAggregation)))); + } + + private BaselineTimeAggregation getBaselineAggregation(TimeAggregation timeAggregation) { + return BaselineTimeAggregation.newBuilder() + .setPeriod(timeAggregation.getPeriod()) + .setAggregation(timeAggregation.getAggregation().getFunction()) + .build(); + } + + private BaselineRequestContext getRequestContext( + String tenantId, + Map requestHeaders, + BaselineEntitiesRequest baselineEntitiesRequest) { + BaselineRequestContext requestContext = new BaselineRequestContext(tenantId, requestHeaders); + baselineEntitiesRequest + .getBaselineMetricSeriesRequestList() + .forEach( + (timeAggregation -> + requestContext.mapAliasToTimeAggregation( + timeAggregation.getAggregation().getAlias(), timeAggregation))); + return requestContext; + } + + private BaselineEntitiesResponse mergeEntities( + Map baselineEntityAggregatedMetricsMap, + Map baselineEntityTimeSeriesMap) { + BaselineEntitiesResponse.Builder baselineEntitiesResponseBuilder = + BaselineEntitiesResponse.newBuilder(); + List baselineEntityList = new ArrayList<>(); + if (!baselineEntityAggregatedMetricsMap.isEmpty()) { + baselineEntityAggregatedMetricsMap.forEach( + (key, value) -> { + BaselineEntity.Builder baselineEntityBuilder = + getBaselineEntityBuilder(value) + .putAllBaselineAggregateMetric(value.getBaselineAggregateMetricMap()); + if (!baselineEntityTimeSeriesMap.isEmpty()) { + baselineEntityBuilder.putAllBaselineMetricSeries( + baselineEntityTimeSeriesMap.get(key).getBaselineMetricSeriesMap()); + } + BaselineEntity baselineEntity = baselineEntityBuilder.build(); + baselineEntityList.add(baselineEntity); + }); + } else if (!baselineEntityTimeSeriesMap.isEmpty()) { + baselineEntityTimeSeriesMap.forEach( + (key, value) -> { + BaselineEntity baselineEntity = + getBaselineEntityBuilder(value) + .putAllBaselineMetricSeries(value.getBaselineMetricSeriesMap()) + .build(); + baselineEntityList.add(baselineEntity); + }); + } + return baselineEntitiesResponseBuilder.addAllBaselineEntity(baselineEntityList).build(); + } + + private Map getEntitiesMapFromTimeSeriesResponse( + BaselineEntitiesResponse baselineEntitiesResponse, + long startTimeInMillis, + long endTimeMillis, + long periodInSecs) { + Map baselineEntityMap = new HashMap<>(); + for (BaselineEntity baselineEntity : baselineEntitiesResponse.getBaselineEntityList()) { + Map metricSeriesMap = + baselineEntity.getBaselineMetricSeriesMap(); + Map baselineMap = new HashMap<>(); + BaselineEntity.Builder baselineEntityBuilder = getBaselineEntityBuilder(baselineEntity); + Map revisedMetricSeriesMap = new HashMap<>(); + // Calculate baseline + metricSeriesMap.forEach( + (key, value) -> { + List metricValues = + value.getBaselineValueList().stream() + .map(x -> x.getBaseline().getValue()) + .collect(Collectors.toList()); + Baseline baseline = BaselineCalculator.getBaseline(metricValues); + baselineMap.put(key, baseline); + }); + // Update intervals + metricSeriesMap.forEach( + (key, value) -> { + List baselineIntervalList = new ArrayList<>(); + long intervalTime = startTimeInMillis; + while (intervalTime < endTimeMillis) { + long periodInMillis = TimeUnit.SECONDS.toMillis(periodInSecs); + BaselineInterval newBaselineInterval = + BaselineInterval.newBuilder() + .setBaseline(baselineMap.get(key)) + .setStartTimeMillis(intervalTime) + .setEndTimeMillis(intervalTime + periodInMillis) + .build(); + baselineIntervalList.add(newBaselineInterval); + intervalTime += periodInMillis; + } + BaselineMetricSeries baselineMetricSeries = + BaselineMetricSeries.newBuilder().addAllBaselineValue(baselineIntervalList).build(); + revisedMetricSeriesMap.put(key, baselineMetricSeries); + }); + baselineEntityBuilder.putAllBaselineMetricSeries(revisedMetricSeriesMap); + baselineEntityMap.put(baselineEntity.getId(), baselineEntityBuilder.build()); + } + return baselineEntityMap; + } + + private BaselineEntity.Builder getBaselineEntityBuilder(BaselineEntity baselineEntity) { + return BaselineEntity.newBuilder() + .setEntityType(baselineEntity.getEntityType()) + .setId(baselineEntity.getId()); + } + + private Map getEntitiesMapFromAggResponse( + BaselineEntitiesResponse baselineEntitiesResponse) { + Map baselineEntityMap = new HashMap<>(); + if (baselineEntitiesResponse.getBaselineEntityCount() > 0) { + for (BaselineEntity baselineEntity : baselineEntitiesResponse.getBaselineEntityList()) { + Map metricSeriesMap = + baselineEntity.getBaselineMetricSeriesMap(); + Map baselineMap = new HashMap<>(); + BaselineEntity.Builder baselineEntityBuilder = getBaselineEntityBuilder(baselineEntity); + metricSeriesMap.forEach( + (key, value) -> { + List metricValues = + value.getBaselineValueList().stream() + .map(x -> x.getBaseline().getValue()) + .collect(Collectors.toList()); + Baseline baseline = BaselineCalculator.getBaseline(metricValues); + baselineMap.put(key, baseline); + }); + baselineEntityBuilder.putAllBaselineAggregateMetric(baselineMap); + baselineEntityMap.put(baselineEntity.getId(), baselineEntityBuilder.build()); + } + } + return baselineEntityMap; + } + + private List getTimeAggregationsForTimeSeriesExpr( + BaselineEntitiesRequest originalRequest) { + List timeSeriesList = + originalRequest.getBaselineMetricSeriesRequestList(); + List timeAggregations = new ArrayList<>(); + for (BaselineTimeAggregation timeAggregation : timeSeriesList) { + Expression aggregation = + Expression.newBuilder().setFunction(timeAggregation.getAggregation()).build(); + TimeAggregation agg = + TimeAggregation.newBuilder() + .setAggregation(aggregation) + .setPeriod(timeAggregation.getPeriod()) + .build(); + timeAggregations.add(agg); + } + return timeAggregations; + } + + private List getTimeAggregationsForAggregateExpr( + BaselineEntitiesRequest originalRequest) { + List aggregateList = originalRequest.getBaselineAggregateRequestList(); + List timeAggregationList = new ArrayList<>(); + for (FunctionExpression function : aggregateList) { + TimeAggregation timeAggregation = + getAggregationFunction( + function, originalRequest.getStartTimeMillis(), originalRequest.getEndTimeMillis()); + timeAggregationList.add(timeAggregation); + } + return timeAggregationList; + } + + private TimeAggregation getAggregationFunction( + FunctionExpression function, long startTimeMillis, long endTimeMillis) { + Period period = getPeriod(startTimeMillis, endTimeMillis); + return TimeAggregation.newBuilder() + .setPeriod(period) + .setAggregation(Expression.newBuilder().setFunction(function).build()) + .build(); + } + + private Period getPeriod(long startTimeInMillis, long endTimeMillis) { + long timePeriod = (endTimeMillis - startTimeInMillis) / (1000 * 60); + return Period.newBuilder().setUnit("MINUTES").setValue((int) timePeriod).build(); + } + + private Period getTimeSeriesPeriod(List baselineTimeSeriesRequestList) { + return baselineTimeSeriesRequestList.get(0).getPeriod(); + } + + private long getPeriodInSecs(Period period) { + ChronoUnit unit = ChronoUnit.valueOf(period.getUnit()); + return Duration.of(period.getValue(), unit).getSeconds(); + } + + /** + * User selected time range(USTR) is difference between end time and start time. It updates start + * time -> startTime minus maximum of (24h, 2xUSTR) + */ + @VisibleForTesting + long getUpdatedStartTime(long startTimeMillis, long endTimeMillis) { + long timeDiff = Math.max(DAY_IN_MILLIS, (2 * (endTimeMillis - startTimeMillis))); + return startTimeMillis - timeDiff; + } +} diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryExecutor.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryExecutor.java new file mode 100644 index 00000000..b21016de --- /dev/null +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryExecutor.java @@ -0,0 +1,24 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.ResultSetChunk; +import org.hypertrace.core.query.service.client.QueryServiceClient; + +import java.util.Iterator; +import java.util.Map; + +public class BaselineServiceQueryExecutor { + + private final int qsRequestTimeout; + private final QueryServiceClient queryServiceClient; + + public BaselineServiceQueryExecutor(int qsRequestTimeout, QueryServiceClient queryServiceClient) { + this.qsRequestTimeout = qsRequestTimeout; + this.queryServiceClient = queryServiceClient; + } + + public Iterator executeQuery( + Map requestHeaders, QueryRequest aggQueryRequest) { + return queryServiceClient.executeQuery(aggQueryRequest, requestHeaders, qsRequestTimeout); + } +} diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParser.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParser.java new file mode 100644 index 00000000..8d41bed8 --- /dev/null +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParser.java @@ -0,0 +1,212 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.core.query.service.api.ColumnMetadata; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.ResultSetChunk; +import org.hypertrace.core.query.service.api.ResultSetMetadata; +import org.hypertrace.core.query.service.api.Row; +import org.hypertrace.core.query.service.client.QueryServiceClient; +import org.hypertrace.gateway.service.common.converters.QueryRequestUtil; +import org.hypertrace.gateway.service.common.AttributeMetadataProvider; +import org.hypertrace.gateway.service.common.converters.QueryAndGatewayDtoConverter; +import org.hypertrace.gateway.service.common.util.MetricAggregationFunctionUtil; +import org.hypertrace.gateway.service.entity.EntityKey; +import org.hypertrace.gateway.service.v1.baseline.Baseline; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntity; +import org.hypertrace.gateway.service.v1.baseline.BaselineInterval; +import org.hypertrace.gateway.service.v1.baseline.BaselineMetricSeries; +import org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation; +import org.hypertrace.gateway.service.v1.common.TimeAggregation; +import org.hypertrace.gateway.service.v1.common.Value; +import org.hypertrace.gateway.service.v1.common.ValueType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createFilter; +import static org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createStringArrayLiteralExpression; +import static org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createStringNullLiteralExpression; +import static org.hypertrace.gateway.service.common.converters.QueryRequestUtil.createTimeColumnGroupByExpression; + +public class BaselineServiceQueryParser { + private static final Logger LOG = LoggerFactory.getLogger(BaselineServiceQueryParser.class); + private final AttributeMetadataProvider attributeMetadataProvider; + + public BaselineServiceQueryParser(AttributeMetadataProvider attributeMetadataProvider) { + this.attributeMetadataProvider = attributeMetadataProvider; + } + + public QueryRequest getQueryRequest( + long startTimeInMillis, + long endTimeInMillis, + List entityIds, + String timeColumn, + List timeAggregationList, + long periodSecs, + List entityIdAttributes) { + QueryRequest.Builder builder = QueryRequest.newBuilder(); + timeAggregationList.forEach( + e -> + builder.addSelection( + QueryAndGatewayDtoConverter.convertToQueryExpression(e.getAggregation()))); + + Filter.Builder queryFilter = + constructQueryServiceFilter( + startTimeInMillis, endTimeInMillis, entityIdAttributes, timeColumn, entityIds); + builder.setFilter(queryFilter); + + builder.addAllGroupBy( + entityIdAttributes.stream() + .map(QueryRequestUtil::createColumnExpression) + .collect(Collectors.toList())); + + builder.addGroupBy(createTimeColumnGroupByExpression(timeColumn, periodSecs)); + + builder.setLimit(QueryServiceClient.DEFAULT_QUERY_SERVICE_GROUP_BY_LIMIT); + + return builder.build(); + } + + private Filter.Builder constructQueryServiceFilter( + long startTimeInMillis, + long endTimeInMillis, + List entityIdAttributes, + String timeColumn, + List entityIds) { + // adds the Id != "null" filter to remove null entities. + Filter.Builder filterBuilder = + Filter.newBuilder() + .setOperator(Operator.AND) + .addAllChildFilter( + entityIdAttributes.stream() + .map( + entityIdAttribute -> + createFilter( + entityIdAttribute, + Operator.NEQ, + createStringNullLiteralExpression())) + .collect(Collectors.toList())); + + filterBuilder.addAllChildFilter( + entityIdAttributes.stream() + .map( + entityIdAttribute -> + QueryRequestUtil.createFilter( + entityIdAttribute, + Operator.IN, + createStringArrayLiteralExpression(entityIds))) + .collect(Collectors.toList())); + + // Time range is a mandatory filter for query service, hence add it if it's not already present. + filterBuilder.addChildFilter( + QueryRequestUtil.createBetweenTimesFilter(timeColumn, startTimeInMillis, endTimeInMillis)); + + return filterBuilder; + } + + public BaselineEntitiesResponse parseQueryResponse( + Iterator resultSetChunkIterator, + BaselineRequestContext requestContext, + int idColumnsSize, + String entityType, + long startTime, + long endTime) { + Map attributeMetadataMap = + attributeMetadataProvider.getAttributesMetadata(requestContext, entityType); + Map> entityMetricSeriesMap = + new LinkedHashMap<>(); + boolean isFirstChunk = true; + ResultSetMetadata resultMetadata = null; + while (resultSetChunkIterator.hasNext()) { + ResultSetChunk chunk = resultSetChunkIterator.next(); + LOG.debug("Received chunk: {} ", chunk); + if (chunk.getRowCount() < 1) { + break; + } + if (isFirstChunk) { + resultMetadata = chunk.getResultSetMetadata(); + isFirstChunk = false; + } + + for (Row row : chunk.getRowList()) { + EntityKey entityKey = + EntityKey.of( + IntStream.range(0, idColumnsSize) + .mapToObj(value -> row.getColumn(value).getString()) + .toArray(String[]::new)); + + Map metricSeriesMap = + entityMetricSeriesMap.computeIfAbsent(entityKey, k -> new LinkedHashMap<>()); + + BaselineInterval.Builder intervalBuilder = BaselineInterval.newBuilder(); + + Value value = + QueryAndGatewayDtoConverter.convertQueryValueToGatewayValue( + row.getColumn(idColumnsSize)); + if (value.getValueType() == ValueType.STRING) { + for (int i = idColumnsSize + 1; + i < resultMetadata.getColumnMetadataCount(); + i++) { + ColumnMetadata metadata = resultMetadata.getColumnMetadata(i); + BaselineTimeAggregation timeAggregation = + requestContext.getTimeAggregationByAlias(metadata.getColumnName()); + + if (timeAggregation == null) { + LOG.warn("Couldn't find an aggregate for column: {}", metadata.getColumnName()); + continue; + } + + Value convertedValue = MetricAggregationFunctionUtil.getValueFromFunction(startTime, endTime, attributeMetadataMap, + row.getColumn(i), metadata, timeAggregation.getAggregation()); + + BaselineMetricSeries.Builder seriesBuilder = + metricSeriesMap.computeIfAbsent( + metadata.getColumnName(), k -> BaselineMetricSeries.newBuilder()); + seriesBuilder.addBaselineValue( + BaselineInterval.newBuilder(intervalBuilder.build()) + .setBaseline(Baseline.newBuilder().setValue(convertedValue).build()) + .build()); + } + } else { + LOG.warn( + "Was expecting STRING values only but received valueType: {}", value.getValueType()); + } + } + } + + List baselineEntities = new ArrayList<>(); + for (Map.Entry> entry : + entityMetricSeriesMap.entrySet()) { + BaselineEntity.Builder entityBuilder = + BaselineEntity.newBuilder() + .setEntityType(entityType) + .setId(entry.getKey().toString()) + .putAllBaselineMetricSeries( + entry.getValue().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, e -> getSortedMetricSeries(e.getValue())))); + baselineEntities.add(entityBuilder.build()); + } + return BaselineEntitiesResponse.newBuilder().addAllBaselineEntity(baselineEntities).build(); + } + + + private BaselineMetricSeries getSortedMetricSeries(BaselineMetricSeries.Builder builder) { + List sortedIntervals = new ArrayList<>(builder.getBaselineValueList()); + sortedIntervals.sort(Comparator.comparingLong(BaselineInterval::getStartTimeMillis)); + return BaselineMetricSeries.newBuilder().addAllBaselineValue(sortedIntervals).build(); + } +} diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/converters/QueryRequestUtil.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/converters/QueryRequestUtil.java index 25e62234..80ee3da2 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/converters/QueryRequestUtil.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/converters/QueryRequestUtil.java @@ -16,6 +16,8 @@ */ public class QueryRequestUtil { + public static final String DATE_TIME_CONVERTER = "dateTimeConvert"; + public static Filter createBetweenTimesFilter(String columnName, long lower, long higher) { return Filter.newBuilder() .setOperator(Operator.AND) @@ -109,7 +111,7 @@ public static Expression createTimeColumnGroupByExpression(String timeColumn, lo return Expression.newBuilder() .setFunction( Function.newBuilder() - .setFunctionName("dateTimeConvert") + .setFunctionName(DATE_TIME_CONVERTER) .addArguments(createColumnExpression(timeColumn)) .addArguments(createStringLiteralExpression("1:MILLISECONDS:EPOCH")) .addArguments(createStringLiteralExpression("1:MILLISECONDS:EPOCH")) diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/EntityDataServiceEntityFetcher.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/EntityDataServiceEntityFetcher.java index c1794788..a2c41228 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/EntityDataServiceEntityFetcher.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/EntityDataServiceEntityFetcher.java @@ -120,6 +120,7 @@ public EntityFetcherResponse getEntities( .toArray(String[]::new)); Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder()); entityBuilder.setEntityType(entitiesRequest.getEntityType()); + entityBuilder.setId(entityKey.toString()); // Always include the id in entity since that's needed to make follow up queries in // optimal fashion. If this wasn't really requested by the client, it should be removed diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcher.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcher.java index 9a9a9559..a1c3c715 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcher.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcher.java @@ -134,7 +134,7 @@ public EntityFetcherResponse getEntities( .toArray(String[]::new)); Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder()); entityBuilder.setEntityType(entitiesRequest.getEntityType()); - + entityBuilder.setId(entityKey.toString()); // Always include the id in entity since that's needed to make follow up queries in // optimal fashion. If this wasn't really requested by the client, it should be removed // as post processing. @@ -225,7 +225,7 @@ public EntityFetcherResponse getAggregatedMetrics( .toArray(String[]::new)); Builder entityBuilder = entityMap.computeIfAbsent(entityKey, k -> Entity.newBuilder()); entityBuilder.setEntityType(entitiesRequest.getEntityType()); - + entityBuilder.setId(entityKey.toString()); // Always include the id in entity since that's needed to make follow up queries in // optimal fashion. If this wasn't really requested by the client, it should be removed // as post processing. @@ -310,6 +310,7 @@ public EntityFetcherResponse getEntitiesAndAggregatedMetrics( .toArray(String[]::new)); Builder entityBuilder = entityBuilders.computeIfAbsent(entityKey, k -> Entity.newBuilder()); entityBuilder.setEntityType(entitiesRequest.getEntityType()); + entityBuilder.setId(entityKey.toString()); // Always include the id in entity since that's needed to make follow up queries in // optimal fashion. If this wasn't really requested by the client, it should be removed @@ -459,37 +460,22 @@ private void addAggregateMetric(Entity.Builder entityBuilder, Preconditions.checkArgument(healthExpressions.size() <= 1); Health health = Health.NOT_COMPUTED; - if (FunctionType.AVGRATE == function.getFunction()) { - Value avgRateValue = - ArithmeticValueUtil.computeAvgRate( - function, - columnValue, - entitiesRequest.getStartTimeMillis(), - entitiesRequest.getEndTimeMillis()); - - entityBuilder.putMetric( - metadata.getColumnName(), - AggregatedMetricValue.newBuilder() - .setValue(avgRateValue) - .setFunction(function.getFunction()) - .setHealth(health) - .build()); - } else { - Value gwValue = - QueryAndGatewayDtoConverter.convertToGatewayValueForMetricValue( - MetricAggregationFunctionUtil.getValueTypeFromFunction( - function, attributeMetadataMap), - attributeMetadataMap, - metadata, - columnValue); - entityBuilder.putMetric( - metadata.getColumnName(), - AggregatedMetricValue.newBuilder() - .setValue(gwValue) - .setFunction(function.getFunction()) - .setHealth(health) - .build()); - } + Value convertedValue = + MetricAggregationFunctionUtil.getValueFromFunction( + entitiesRequest.getStartTimeMillis(), + entitiesRequest.getEndTimeMillis(), + attributeMetadataMap, + columnValue, + metadata, + function); + + entityBuilder.putMetric( + metadata.getColumnName(), + AggregatedMetricValue.newBuilder() + .setValue(convertedValue) + .setFunction(function.getFunction()) + .setHealth(health) + .build()); } @Override @@ -635,6 +621,7 @@ public EntityFetcherResponse getTimeAggregatedMetrics( Entity.Builder entityBuilder = Entity.newBuilder() .setEntityType(entitiesRequest.getEntityType()) + .setId(entry.getKey().toString()) .putAllMetricSeries( entry.getValue().entrySet().stream() .collect( diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/util/MetricAggregationFunctionUtil.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/util/MetricAggregationFunctionUtil.java index 1971d5f3..5244088d 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/util/MetricAggregationFunctionUtil.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/common/util/MetricAggregationFunctionUtil.java @@ -9,10 +9,13 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.hypertrace.core.attribute.service.v1.AttributeKind; import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.core.query.service.api.ColumnMetadata; +import org.hypertrace.gateway.service.common.converters.QueryAndGatewayDtoConverter; import org.hypertrace.gateway.service.v1.common.Expression; import org.hypertrace.gateway.service.v1.common.Expression.ValueCase; import org.hypertrace.gateway.service.v1.common.FunctionExpression; import org.hypertrace.gateway.service.v1.common.FunctionType; +import org.hypertrace.gateway.service.v1.common.Value; /** * Class with some utility methods around Aggregated metrics, alias in the entity requests. @@ -113,4 +116,29 @@ public static AttributeKind getValueTypeFromFunction( return metadata.getValueKind(); } } + + public static Value getValueFromFunction( + long startTime, + long endTime, + Map attributeMetadataMap, + org.hypertrace.core.query.service.api.Value column, + ColumnMetadata metadata, + FunctionExpression functionExpression) { + // AVG_RATE is adding a specific implementation because Pinot does not directly support this function, + // so it has to be parsed separately. + Value convertedValue; + if (FunctionType.AVGRATE == functionExpression.getFunction()) { + convertedValue = + ArithmeticValueUtil.computeAvgRate(functionExpression, column, startTime, endTime); + } else { + convertedValue = + QueryAndGatewayDtoConverter.convertToGatewayValueForMetricValue( + MetricAggregationFunctionUtil.getValueTypeFromFunction( + functionExpression, attributeMetadataMap), + attributeMetadataMap, + metadata, + column); + } + return convertedValue; + } } diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/EntityService.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/EntityService.java index 5072d6e8..cce3cc71 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/EntityService.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/entity/EntityService.java @@ -153,7 +153,9 @@ public EntitiesResponse getEntities( EntitiesResponse.Builder responseBuilder = EntitiesResponse.newBuilder().setTotal(executionContext.getTotal()); - results.forEach(e -> responseBuilder.addEntity(e.build())); + results.forEach(e -> { + responseBuilder.addEntity(e.build()); + }); long queryExecutionTime = System.currentTimeMillis() - startTime; if (queryExecutionTime > logConfig.getQueryThresholdInMillis()) { diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java index edd4ca4d..36d15b3f 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/RequestHandler.java @@ -284,8 +284,13 @@ void handleQueryServiceResponseSingleColumn( org.hypertrace.gateway.service.v1.common.Value gwValue; if (function != null) { // Function expression value gwValue = - getValueForFunctionExpression( - queryServiceValue, function, requestContext, metadata, attributeMetadataMap); + MetricAggregationFunctionUtil.getValueFromFunction( + requestContext.getStartTimeMillis(), + requestContext.getEndTimeMillis(), + attributeMetadataMap, + queryServiceValue, + metadata, + function); } else { // Simple columnId Expression value eg. groupBy columns or column selections gwValue = getValueForColumnIdExpression(queryServiceValue, metadata, attributeMetadataMap); } @@ -293,26 +298,6 @@ void handleQueryServiceResponseSingleColumn( rowBuilder.putColumns(metadata.getColumnName(), gwValue); } - private org.hypertrace.gateway.service.v1.common.Value getValueForFunctionExpression( - Value queryServiceValue, - org.hypertrace.gateway.service.v1.common.FunctionExpression function, - ExploreRequestContext requestContext, - ColumnMetadata metadata, - Map attributeMetadataMap) { - if (FunctionType.AVGRATE == function.getFunction()) { - return ArithmeticValueUtil.computeAvgRate( - function, - queryServiceValue, - requestContext.getStartTimeMillis(), - requestContext.getEndTimeMillis()); - } else { - return QueryAndGatewayDtoConverter.convertToGatewayValueForMetricValue( - MetricAggregationFunctionUtil.getValueTypeFromFunction(function, attributeMetadataMap), - attributeMetadataMap, - metadata, - queryServiceValue); - } - } private org.hypertrace.gateway.service.v1.common.Value getValueForColumnIdExpression( Value queryServiceValue, diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineCalculatorTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineCalculatorTest.java new file mode 100644 index 00000000..77752b2e --- /dev/null +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineCalculatorTest.java @@ -0,0 +1,48 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.gateway.service.v1.baseline.Baseline; +import org.hypertrace.gateway.service.v1.common.Value; +import org.hypertrace.gateway.service.v1.common.ValueType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +public class BaselineCalculatorTest { + + @Test + public void testBaselineForDoubleValues() { + List values = new ArrayList<>(); + values.add(Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(2).build()); + values.add(Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(10).build()); + values.add(Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(100).build()); + Baseline baseline = BaselineCalculator.getBaseline(values); + Assertions.assertEquals(10.0, baseline.getValue().getDouble()); + // Lower bound will be negative so it should be zero. + Assertions.assertEquals(0, baseline.getLowerBound().getDouble()); + } + + @Test + public void testBaselineForLongValues() { + List values = new ArrayList<>(); + values.add(Value.newBuilder().setValueType(ValueType.LONG).setLong(8).build()); + values.add(Value.newBuilder().setValueType(ValueType.LONG).setLong(10).build()); + values.add(Value.newBuilder().setValueType(ValueType.LONG).setLong(12).build()); + Baseline baseline = BaselineCalculator.getBaseline(values); + Assertions.assertEquals(10.0, baseline.getValue().getDouble()); + Assertions.assertEquals(6.0, baseline.getLowerBound().getDouble()); + } + + @Test + public void testBaselineUnsupportedType() { + List values = new ArrayList<>(); + values.add(Value.newBuilder().setValueType(ValueType.STRING).setString("2").build()); + values.add(Value.newBuilder().setValueType(ValueType.STRING).setString("abc").build()); + values.add(Value.newBuilder().setValueType(ValueType.STRING).setString("s34").build()); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> BaselineCalculator.getBaseline(values), + "Baseline cannot be calculated for String values"); + } +} diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidatorTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidatorTest.java new file mode 100644 index 00000000..c8780750 --- /dev/null +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineEntitiesRequestValidatorTest.java @@ -0,0 +1,71 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class BaselineEntitiesRequestValidatorTest { + @Test + public void emptyEntityIds_shouldThrowIllegalArgException() { + Map attributeMetadataMap = new HashMap<>(); + BaselineEntitiesRequest baselineEntitiesRequest = + BaselineEntitiesRequest.newBuilder() + .setEntityType("API") + .addAllEntityIds(Collections.emptyList()) + .setStartTimeMillis(Instant.parse("2020-11-14T17:40:51.902Z").toEpochMilli()) + .setEndTimeMillis(Instant.parse("2020-11-14T18:40:51.902Z").toEpochMilli()) + .build(); + BaselineEntitiesRequestValidator baselineEntitiesRequestValidator = + new BaselineEntitiesRequestValidator(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + baselineEntitiesRequestValidator.validate(baselineEntitiesRequest, attributeMetadataMap); + }); + } + + @Test + public void emptyAggregationsAndTimeSeries_shouldThrowIllegalArgException() { + Map attributeMetadataMap = new HashMap<>(); + BaselineEntitiesRequest baselineEntitiesRequest = + BaselineEntitiesRequest.newBuilder() + .setEntityType("API") + .addAllEntityIds(Collections.singleton("entity1")) + .setStartTimeMillis(Instant.parse("2020-11-14T17:40:51.902Z").toEpochMilli()) + .setEndTimeMillis(Instant.parse("2020-11-14T18:40:51.902Z").toEpochMilli()) + .addAllBaselineAggregateRequest(Collections.emptyList()) + .addAllBaselineMetricSeriesRequest(Collections.emptyList()) + .build(); + BaselineEntitiesRequestValidator baselineEntitiesRequestValidator = + new BaselineEntitiesRequestValidator(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + baselineEntitiesRequestValidator.validate(baselineEntitiesRequest, attributeMetadataMap); + }); + } + + @Test + public void entityTypeNotPresent_shouldThrowIllegalArgException() { + Map attributeMetadataMap = new HashMap<>(); + BaselineEntitiesRequest baselineEntitiesRequest = + BaselineEntitiesRequest.newBuilder() + .addAllEntityIds(Collections.singleton("entity1")) + .setStartTimeMillis(Instant.parse("2020-11-14T17:40:51.902Z").toEpochMilli()) + .setEndTimeMillis(Instant.parse("2020-11-14T18:40:51.902Z").toEpochMilli()) + .build(); + BaselineEntitiesRequestValidator baselineEntitiesRequestValidator = + new BaselineEntitiesRequestValidator(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + baselineEntitiesRequestValidator.validate(baselineEntitiesRequest, attributeMetadataMap); + }); + } +} diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceImplTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceImplTest.java new file mode 100644 index 00000000..89c0006a --- /dev/null +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceImplTest.java @@ -0,0 +1,195 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.ResultSetChunk; +import org.hypertrace.gateway.service.common.AttributeMetadataProvider; +import org.hypertrace.gateway.service.common.RequestContext; +import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs; +import org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation; +import org.hypertrace.gateway.service.v1.common.ColumnIdentifier; +import org.hypertrace.gateway.service.v1.common.Expression; +import org.hypertrace.gateway.service.v1.common.FunctionExpression; +import org.hypertrace.gateway.service.v1.common.FunctionType; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse; +import org.hypertrace.gateway.service.v1.common.Period; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.hypertrace.gateway.service.baseline.BaselineServiceImpl.DAY_IN_MILLIS; +import static org.hypertrace.gateway.service.common.QueryServiceRequestAndResponseUtils.getResultSetChunk; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BaselineServiceImplTest { + + protected static final String TENANT_ID = "tenant1"; + private static final long ONE_HOUR_MILLIS = 1000 * 60 * 60; + private static final long TWENTY_HOUR_MILLIS = 20 * ONE_HOUR_MILLIS; + private final AttributeMetadataProvider attributeMetadataProvider = + Mockito.mock(AttributeMetadataProvider.class); + private final BaselineServiceQueryExecutor baselineServiceQueryExecutor = + Mockito.mock(BaselineServiceQueryExecutor.class); + private final BaselineServiceQueryParser baselineServiceQueryParser = + new BaselineServiceQueryParser(attributeMetadataProvider); + private final EntityIdColumnsConfigs entityIdColumnsConfigs = mock(EntityIdColumnsConfigs.class); + + @Test + public void testBaselineForEntitiesForAggregates() { + BaselineEntitiesRequest baselineEntitiesRequest = + BaselineEntitiesRequest.newBuilder() + .setEntityType("SERVICE") + .setStartTimeMillis(Instant.parse("2020-11-14T17:40:51.902Z").toEpochMilli()) + .setEndTimeMillis(Instant.parse("2020-11-14T18:40:51.902Z").toEpochMilli()) + .addEntityIds("entity-1") + .addBaselineAggregateRequest( + getFunctionExpressionFor(FunctionType.AVG, "SERVICE.duration", "duration_ts")) + .build(); + + // Mock section + AttributeMetadata attributeMetadata = + AttributeMetadata.newBuilder().setFqn("Service.Latency").setId("Service.StartTime").build(); + Mockito.when( + attributeMetadataProvider.getAttributeMetadata( + Mockito.any(RequestContext.class), Mockito.anyString(), Mockito.anyString())) + .thenReturn(Optional.of(attributeMetadata)); + Mockito.when( + baselineServiceQueryExecutor.executeQuery( + Mockito.anyMap(), Mockito.any(QueryRequest.class))) + .thenReturn(getResultSet().iterator()); + when(entityIdColumnsConfigs.getIdKey("SERVICE")).thenReturn(Optional.of("id")); + + Map attributeMap = new HashMap<>(); + attributeMap.put( + "duration_ts", + AttributeMetadata.newBuilder().setFqn("Service.Latency").setId("Service.Id").build()); + attributeMap.put( + "SERVICE.duration", + AttributeMetadata.newBuilder().setFqn("Service.Latency").setId("Service.Id").build()); + Mockito.when( + attributeMetadataProvider.getAttributesMetadata( + Mockito.any(RequestContext.class), Mockito.anyString())) + .thenReturn(attributeMap); + + BaselineService baselineService = + new BaselineServiceImpl( + attributeMetadataProvider, + baselineServiceQueryParser, + baselineServiceQueryExecutor, + entityIdColumnsConfigs); + BaselineEntitiesResponse baselineResponse = + baselineService.getBaselineForEntities(TENANT_ID, baselineEntitiesRequest, Map.of()); + Assertions.assertTrue(baselineResponse.getBaselineEntityCount() > 0); + Assertions.assertTrue( + baselineResponse.getBaselineEntityList().get(0).getBaselineAggregateMetricCount() > 0); + } + + @Test + public void testBaselineEntitiesForMetricSeries() { + BaselineEntitiesRequest baselineEntitiesRequest = + BaselineEntitiesRequest.newBuilder() + .setEntityType("SERVICE") + .setStartTimeMillis(Instant.parse("2020-11-14T17:40:51.902Z").toEpochMilli()) + .setEndTimeMillis(Instant.parse("2020-11-14T18:40:51.902Z").toEpochMilli()) + .addEntityIds("entity-1") + .addBaselineMetricSeriesRequest(getBaselineTimeSeriesRequest()) + .build(); + // Mock section + AttributeMetadata attributeMetadata = + AttributeMetadata.newBuilder().setFqn("Service.Latency").setId("Service.StartTime").build(); + Mockito.when( + attributeMetadataProvider.getAttributeMetadata( + Mockito.any(RequestContext.class), Mockito.anyString(), Mockito.anyString())) + .thenReturn(Optional.of(attributeMetadata)); + Mockito.when( + baselineServiceQueryExecutor.executeQuery( + Mockito.anyMap(), Mockito.any(QueryRequest.class))) + .thenReturn(getResultSet().iterator()); + Map attributeMap = new HashMap<>(); + AttributeMetadata attribute = + AttributeMetadata.newBuilder().setFqn("Service.Latency").setId("Service.Id").build(); + attributeMap.put("duration_ts", attribute); + attributeMap.put("SERVICE.duration", attribute); + Mockito.when( + attributeMetadataProvider.getAttributesMetadata( + Mockito.any(RequestContext.class), Mockito.anyString())) + .thenReturn(attributeMap); + Mockito.when( + attributeMetadataProvider.getAttributeMetadata( + Mockito.any(RequestContext.class), Mockito.anyString(), Mockito.anyString())) + .thenReturn(Optional.of(attribute)); + when(entityIdColumnsConfigs.getIdKey("SERVICE")).thenReturn(Optional.of("id")); + + BaselineService baselineService = + new BaselineServiceImpl( + attributeMetadataProvider, + baselineServiceQueryParser, + baselineServiceQueryExecutor, + entityIdColumnsConfigs); + BaselineEntitiesResponse baselineResponse = + baselineService.getBaselineForEntities(TENANT_ID, baselineEntitiesRequest, Map.of()); + Assertions.assertTrue(baselineResponse.getBaselineEntityCount() > 0); + Assertions.assertTrue( + baselineResponse.getBaselineEntityList().get(0).getBaselineMetricSeriesCount() > 0); + } + + private BaselineTimeAggregation getBaselineTimeSeriesRequest() { + return BaselineTimeAggregation.newBuilder() + .setAggregation( + getFunctionExpressionFor(FunctionType.AVG, "SERVICE.duration", "duration_ts")) + .setPeriod(Period.newBuilder().setUnit("MINUTES").setValue(1).build()) + .build(); + } + + private FunctionExpression getFunctionExpressionFor( + FunctionType type, String columnName, String alias) { + return FunctionExpression.newBuilder() + .setFunction(type) + .setAlias(alias) + .addArguments( + Expression.newBuilder() + .setColumnIdentifier( + ColumnIdentifier.newBuilder().setColumnName(columnName).setAlias(alias))) + .build(); + } + + public List getResultSet() { + long time = System.currentTimeMillis(); + + List resultSetChunks = + List.of( + getResultSetChunk( + List.of("SERVICE.id", "dateTimeConvert", "duration_ts"), + new String[][] { + {"entity-1", String.valueOf(time), "14.0"}, + {"entity-1", String.valueOf(time - 60000), "15.0"}, + {"entity-1", String.valueOf(time - 120000), "16.0"}, + {"entity-1", String.valueOf(time - 180000), "17.0"} + })); + return resultSetChunks; + } + + @Test + public void testStartTimeCalcGivenTimeRange() { + BaselineServiceImpl baselineService = + new BaselineServiceImpl( + attributeMetadataProvider, baselineServiceQueryParser, + baselineServiceQueryExecutor, entityIdColumnsConfigs); + long endTimeInMillis = System.currentTimeMillis(); + long startTimeInMillis = endTimeInMillis - ONE_HOUR_MILLIS; + long actualStartTime = baselineService.getUpdatedStartTime(startTimeInMillis, endTimeInMillis); + Assertions.assertEquals(startTimeInMillis - DAY_IN_MILLIS, actualStartTime); + + startTimeInMillis = endTimeInMillis - TWENTY_HOUR_MILLIS; + actualStartTime = baselineService.getUpdatedStartTime(startTimeInMillis, endTimeInMillis); + Assertions.assertEquals(startTimeInMillis - (2 * TWENTY_HOUR_MILLIS), actualStartTime); + } +} diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParserTest.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParserTest.java new file mode 100644 index 00000000..b6ebd0bb --- /dev/null +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/baseline/BaselineServiceQueryParserTest.java @@ -0,0 +1,131 @@ +package org.hypertrace.gateway.service.baseline; + +import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.ResultSetChunk; +import org.hypertrace.gateway.service.common.AttributeMetadataProvider; +import org.hypertrace.gateway.service.common.RequestContext; +import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse; +import org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation; +import org.hypertrace.gateway.service.v1.common.ColumnIdentifier; +import org.hypertrace.gateway.service.v1.common.Expression; +import org.hypertrace.gateway.service.v1.common.FunctionExpression; +import org.hypertrace.gateway.service.v1.common.FunctionType; +import org.hypertrace.gateway.service.v1.common.Period; +import org.hypertrace.gateway.service.v1.common.TimeAggregation; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hypertrace.gateway.service.common.QueryServiceRequestAndResponseUtils.getResultSetChunk; + +public class BaselineServiceQueryParserTest { + + private static final long ONE_HOUR_SECONDS = 60 * 60L; + private final AttributeMetadataProvider attributeMetadataProvider = + Mockito.mock(AttributeMetadataProvider.class); + protected static final String TENANT_ID = "tenant1"; + + @Test + public void testGetQueryRequest() { + BaselineServiceQueryParser baselineServiceQueryParser = + new BaselineServiceQueryParser(attributeMetadataProvider); + List timeAggregationList = new ArrayList<>(); + long periodInSecs = ONE_HOUR_SECONDS; + TimeAggregation timeAggregation = + getTimeAggregationFor( + getFunctionExpressionFor(FunctionType.AVG, "SERVICE.duration", "duration_ts")); + timeAggregationList.add(timeAggregation); + QueryRequest request = + baselineServiceQueryParser.getQueryRequest( + Instant.parse("2020-11-14T17:40:51.902Z").toEpochMilli(), + Instant.parse("2020-11-14T18:40:51.902Z").toEpochMilli(), + Collections.singletonList("entity-1"), + "Service.StartTime", + timeAggregationList, + periodInSecs, + Collections.singletonList("SERVICE.id")); + Assertions.assertNotNull(request); + Assertions.assertEquals(2, request.getGroupByCount()); + Assertions.assertEquals(3, request.getFilter().getChildFilterCount()); + } + + @Test + public void testQueryResponse() { + BaselineServiceQueryParser baselineServiceQueryParser = + new BaselineServiceQueryParser(attributeMetadataProvider); + List resultSetChunks = + List.of( + getResultSetChunk( + List.of("SERVICE.id", "dateTimeConvert", "PERCENTILE_SERVICE.duration_[99]_PT30S"), + new String[][] { + {"entity-id-1", "1608524400000", "14.0"}, + {"entity-id-1", "1608525210000", "15.0"}, + {"entity-id-1", "1608525840000", "16.0"}, + {"entity-id-1", "1608525540000", "17.0"} + })); + TimeAggregation timeAggregation = + getTimeAggregationFor( + getFunctionExpressionFor( + FunctionType.PERCENTILE, + "SERVICE.Latency", + "PERCENTILE_SERVICE.duration_[99]_PT30S")); + BaselineRequestContext baselineRequestContext = + new BaselineRequestContext(TENANT_ID, Collections.EMPTY_MAP); + BaselineTimeAggregation baselineTimeAggregation = + BaselineTimeAggregation.newBuilder() + .setAggregation(timeAggregation.getAggregation().getFunction()) + .setPeriod(timeAggregation.getPeriod()) + .build(); + baselineRequestContext.mapAliasToTimeAggregation( + "PERCENTILE_SERVICE.duration_[99]_PT30S", baselineTimeAggregation); + Map attributeMap = new HashMap<>(); + attributeMap.put( + "SERVICE.Latency", + AttributeMetadata.newBuilder().setFqn("Service.Latency").setId("Service.Id").build()); + Mockito.when( + attributeMetadataProvider.getAttributesMetadata( + Mockito.any(RequestContext.class), Mockito.anyString())) + .thenReturn(attributeMap); + BaselineEntitiesResponse response = + baselineServiceQueryParser.parseQueryResponse( + resultSetChunks.iterator(), + baselineRequestContext, + 1, + "SERVICE", + Instant.parse("2020-11-14T17:40:51.902Z").toEpochMilli(), + Instant.parse("2020-11-14T18:40:51.902Z").toEpochMilli()); + Assertions.assertNotNull(response); + Assertions.assertEquals(1, response.getBaselineEntity(0).getBaselineMetricSeriesCount()); + } + + private TimeAggregation getTimeAggregationFor(Expression expression) { + return TimeAggregation.newBuilder() + .setAggregation(expression) + .setPeriod(Period.newBuilder().setUnit("SECONDS").setValue(60).build()) + .build(); + } + + private Expression getFunctionExpressionFor(FunctionType type, String columnName, String alias) { + return Expression.newBuilder() + .setFunction( + FunctionExpression.newBuilder() + .setFunction(type) + .setAlias(alias) + .addArguments( + Expression.newBuilder() + .setColumnIdentifier( + ColumnIdentifier.newBuilder() + .setColumnName(columnName) + .setAlias(alias))) + .build()) + .build(); + } +} diff --git a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcherTests.java b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcherTests.java index 6c5c5604..79ff98cd 100644 --- a/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcherTests.java +++ b/gateway-service-impl/src/test/java/org/hypertrace/gateway/service/common/datafetcher/QueryServiceEntityFetcherTests.java @@ -154,21 +154,25 @@ public void test_getEntitiesAndAggregatedMetrics() { Map expectedEntityKeyBuilderResponseMap = Map.of( EntityKey.of("api-id-0"), Entity.newBuilder() .setEntityType(AttributeScope.API.name()) + .setId("api-id-0") .putAttribute(API_NAME_ATTR, getStringValue("api-0")) .putAttribute(API_ID_ATTR, getStringValue("api-id-0")) .putMetric("AVG_API.duration", getAggregatedMetricValue(FunctionType.AVG, 14.0)), EntityKey.of("api-id-1"), Entity.newBuilder() .setEntityType(AttributeScope.API.name()) + .setId("api-id-1") .putAttribute(API_NAME_ATTR, getStringValue("api-1")) .putAttribute(API_ID_ATTR, getStringValue("api-id-1")) .putMetric("AVG_API.duration", getAggregatedMetricValue(FunctionType.AVG, 15.0)), EntityKey.of("api-id-2"), Entity.newBuilder() .setEntityType(AttributeScope.API.name()) + .setId("api-id-2") .putAttribute(API_NAME_ATTR, getStringValue("api-2")) .putAttribute(API_ID_ATTR, getStringValue("api-id-2")) .putMetric("AVG_API.duration", getAggregatedMetricValue(FunctionType.AVG, 16.0)), EntityKey.of("api-id-3"), Entity.newBuilder() .setEntityType(AttributeScope.API.name()) + .setId("api-id-3") .putAttribute(API_NAME_ATTR, getStringValue("api-3")) .putAttribute(API_ID_ATTR, getStringValue("api-id-3")) .putMetric("AVG_API.duration", getAggregatedMetricValue(FunctionType.AVG, 17.0)) @@ -301,6 +305,7 @@ public void test_getEntitiesBySpace() { Map expectedEntityKeyBuilderResponseMap = Map.of( EntityKey.of("api-id-0"), Entity.newBuilder() .setEntityType(AttributeScope.API.name()) + .setId("api-id-0") .putAttribute(API_NAME_ATTR, getStringValue("api-0")) .putAttribute(API_ID_ATTR, getStringValue("api-id-0")) .putMetric("AVG_API.duration", getAggregatedMetricValue(FunctionType.AVG, 14.0))