Skip to content
This repository has been archived by the owner on Jun 26, 2024. It is now read-only.

Commit

Permalink
Add baseline api and implementaion for entities (#55)
Browse files Browse the repository at this point in the history
* Add baseline api and implementation for entities
  • Loading branch information
pavan-traceable authored Jan 6, 2021
1 parent f660f94 commit b61884d
Show file tree
Hide file tree
Showing 23 changed files with 1,344 additions and 60 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
syntax = "proto3";

option java_multiple_files = true;
option java_package = "org.hypertrace.gateway.service.v1.baseline";

package org.hypertrace.gateway.service.v1.baseline;

import "org/hypertrace/gateway/service/v1/gateway_query.proto";

message BaselineEntitiesRequest {
string entity_type = 1;
sfixed64 start_time_millis = 2;
sfixed64 end_time_millis = 3;

repeated string entity_ids = 4;
repeated org.hypertrace.gateway.service.v1.common.FunctionExpression baseline_aggregate_request = 7;
repeated BaselineTimeAggregation baseline_metric_series_request = 8;
}

message BaselineEntitiesResponse {
repeated BaselineEntity baseline_entity = 1;
}

message BaselineEntity {
string id = 1;
string entity_type = 2;
map<string, Baseline> baseline_aggregate_metric = 5;
map<string, BaselineMetricSeries> baseline_metric_series = 6;
}

message BaselineTimeAggregation {
org.hypertrace.gateway.service.v1.common.Period period = 1;
org.hypertrace.gateway.service.v1.common.FunctionExpression aggregation = 2;
}

message Baseline {
org.hypertrace.gateway.service.v1.common.Value value = 1;
org.hypertrace.gateway.service.v1.common.Value lower_bound = 2;
org.hypertrace.gateway.service.v1.common.Value upper_bound = 3;
}

message BaselineMetricSeries {
repeated BaselineInterval baseline_value = 1;
}

message BaselineInterval {
sfixed64 start_time_millis = 1;
sfixed64 end_time_millis = 2;
Baseline baseline = 3;
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import "org/hypertrace/gateway/service/v1/gateway_query.proto";

// Entity + metric data
message Entity {
string id = 1;
string entity_type = 2;
map<string, org.hypertrace.gateway.service.v1.common.Value> attribute = 4;
map<string, org.hypertrace.gateway.service.v1.common.AggregatedMetricValue> metric = 5;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,15 @@ import "org/hypertrace/gateway/service/v1/entities.proto";
import "org/hypertrace/gateway/service/v1/explore.proto";
import "org/hypertrace/gateway/service/v1/spans.proto";
import "org/hypertrace/gateway/service/v1/traces.proto";
import "org/hypertrace/gateway/service/v1/baseline.proto";

service GatewayService {
rpc getEntities (org.hypertrace.gateway.service.v1.entity.EntitiesRequest)
returns (org.hypertrace.gateway.service.v1.entity.EntitiesResponse) {}
rpc updateEntity (org.hypertrace.gateway.service.v1.entity.UpdateEntityRequest)
returns (org.hypertrace.gateway.service.v1.entity.UpdateEntityResponse) {}
rpc getBaselineForEntities(org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest)
returns (org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse) {}
rpc explore (org.hypertrace.gateway.service.v1.explore.ExploreRequest)
returns (org.hypertrace.gateway.service.v1.explore.ExploreResponse) {}
rpc getTraces (org.hypertrace.gateway.service.v1.trace.TracesRequest)
Expand Down
1 change: 1 addition & 0 deletions gateway-service-impl/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ dependencies {

// Common utilities
implementation("org.apache.commons:commons-lang3:3.10")
implementation("org.apache.commons:commons-math:2.2")
implementation("com.google.protobuf:protobuf-java-util:3.13.0")

// Metrics
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,26 @@
import org.hypertrace.core.query.service.client.QueryServiceConfig;
import org.hypertrace.entity.query.service.client.EntityQueryServiceClient;
import org.hypertrace.entity.service.client.config.EntityServiceClientConfig;
import org.hypertrace.gateway.service.baseline.BaselineServiceQueryExecutor;
import org.hypertrace.gateway.service.baseline.BaselineServiceQueryParser;
import org.hypertrace.gateway.service.common.AttributeMetadataProvider;
import org.hypertrace.gateway.service.common.RequestContext;
import org.hypertrace.gateway.service.common.config.ScopeFilterConfigs;
import org.hypertrace.gateway.service.entity.EntityService;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
import org.hypertrace.gateway.service.entity.config.LogConfig;
import org.hypertrace.gateway.service.explore.ExploreService;
import org.hypertrace.gateway.service.baseline.BaselineService;
import org.hypertrace.gateway.service.baseline.BaselineServiceImpl;
import org.hypertrace.gateway.service.span.SpanService;
import org.hypertrace.gateway.service.trace.TracesService;
import org.hypertrace.gateway.service.v1.entity.EntitiesResponse;
import org.hypertrace.gateway.service.v1.entity.UpdateEntityRequest;
import org.hypertrace.gateway.service.v1.entity.UpdateEntityResponse;
import org.hypertrace.gateway.service.v1.explore.ExploreRequest;
import org.hypertrace.gateway.service.v1.explore.ExploreResponse;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse;
import org.hypertrace.gateway.service.v1.span.SpansResponse;
import org.hypertrace.gateway.service.v1.trace.TracesResponse;
import org.slf4j.Logger;
Expand All @@ -51,6 +57,7 @@ public class GatewayServiceImpl extends GatewayServiceGrpc.GatewayServiceImplBas
private final SpanService spanService;
private final EntityService entityService;
private final ExploreService exploreService;
private final BaselineService baselineService;

public GatewayServiceImpl(Config appConfig) {
AttributeServiceClientConfig asConfig = AttributeServiceClientConfig.from(appConfig);
Expand Down Expand Up @@ -83,6 +90,9 @@ public GatewayServiceImpl(Config appConfig) {
this.exploreService =
new ExploreService(queryServiceClient, qsRequestTimeout,
attributeMetadataProvider, scopeFilterConfigs);
BaselineServiceQueryParser baselineServiceQueryParser = new BaselineServiceQueryParser(attributeMetadataProvider);
BaselineServiceQueryExecutor baselineServiceQueryExecutor = new BaselineServiceQueryExecutor(qsRequestTimeout, queryServiceClient);
this.baselineService = new BaselineServiceImpl(attributeMetadataProvider, baselineServiceQueryParser, baselineServiceQueryExecutor, entityIdColumnsConfigs);
}

private static int getRequestTimeoutMillis(Config config) {
Expand Down Expand Up @@ -184,9 +194,7 @@ public void getEntities(
org.hypertrace.core.grpcutils.context.RequestContext.CURRENT.get()
.getRequestHeaders());

if (LOG.isDebugEnabled()) {
LOG.debug("Received response: {}", response);
}
LOG.debug("Received response: {}", response);

responseObserver.onNext(response);
responseObserver.onCompleted();
Expand Down Expand Up @@ -229,6 +237,35 @@ public void updateEntity(
}
}

@Override
public void getBaselineForEntities(
BaselineEntitiesRequest request, StreamObserver<BaselineEntitiesResponse> responseObserver) {
Optional<String> tenantId =
org.hypertrace.core.grpcutils.context.RequestContext.CURRENT.get().getTenantId();
if (tenantId.isEmpty()) {
responseObserver.onError(new ServiceException("Tenant id is missing in the request."));
return;
}

try {
BaselineEntitiesResponse response =
baselineService.getBaselineForEntities(
tenantId.get(),
request,
org.hypertrace.core.grpcutils.context.RequestContext.CURRENT
.get()
.getRequestHeaders());

LOG.debug("Received response: {}", response);

responseObserver.onNext(response);
responseObserver.onCompleted();
} catch (Exception e) {
LOG.error("Error while handling entities request: {}.", request, e);
responseObserver.onError(e);
}
}

@Override
public void explore(ExploreRequest request, StreamObserver<ExploreResponse> responseObserver) {
Optional<String> tenantId =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package org.hypertrace.gateway.service.baseline;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
import org.apache.commons.math.stat.descriptive.rank.Median;
import org.hypertrace.gateway.service.v1.baseline.Baseline;
import org.hypertrace.gateway.service.v1.common.Value;
import org.hypertrace.gateway.service.v1.common.ValueType;

import java.util.List;

public class BaselineCalculator {

public static Baseline getBaseline(List<Value> metricValues) {
if (metricValues.isEmpty()) {
return Baseline.getDefaultInstance();
}
double[] values = getValuesInDouble(metricValues);
return getBaseline(values);
}

private static Baseline getBaseline(double[] metricValueArray) {
Median median = new Median();
StandardDeviation standardDeviation = new StandardDeviation();
double medianValue = median.evaluate(metricValueArray);
double sd = standardDeviation.evaluate(metricValueArray);
double lowerBound = medianValue - (2 * sd);
if (lowerBound < 0) {
lowerBound = 0;
}
double upperBound = medianValue + (2 * sd);
Baseline baseline =
Baseline.newBuilder()
.setLowerBound(
Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(lowerBound).build())
.setUpperBound(
Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(upperBound).build())
.setValue(
Value.newBuilder().setValueType(ValueType.DOUBLE).setDouble(medianValue).build())
.build();
return baseline;
}

@VisibleForTesting
private static double[] getValuesInDouble(List<Value> metricValues) {
ValueType valueType = metricValues.get(0).getValueType();
switch (valueType) {
case LONG:
return metricValues.stream().mapToDouble(value -> (double) value.getLong()).toArray();
case DOUBLE:
return metricValues.stream().mapToDouble(value -> value.getDouble()).toArray();
default:
throw new IllegalArgumentException("Unsupported valueType " + valueType);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package org.hypertrace.gateway.service.baseline;

import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.hypertrace.core.attribute.service.v1.AttributeMetadata;
import org.hypertrace.gateway.service.common.validators.request.RequestValidator;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Map;

public class BaselineEntitiesRequestValidator extends RequestValidator<BaselineEntitiesRequest> {
private static final Logger LOG = LoggerFactory.getLogger(BaselineEntitiesRequestValidator.class);

@Override
public void validate(
BaselineEntitiesRequest request, Map<String, AttributeMetadata> attributeMetadataMap) {
Preconditions.checkArgument(
StringUtils.isNotBlank(request.getEntityType()), "EntityType is mandatory in the request.");

Preconditions.checkArgument(request.getEntityIdsCount() > 0, "EntityIds cannot be empty");

Preconditions.checkArgument(
(request.getBaselineAggregateRequestCount() > 0
|| request.getBaselineMetricSeriesRequestCount() > 0),
"Both Selection list and TimeSeries list can't be empty in the request.");

Preconditions.checkArgument(
request.getStartTimeMillis() > 0
&& request.getEndTimeMillis() > 0
&& request.getStartTimeMillis() < request.getEndTimeMillis(),
"Invalid time range. Both start and end times have to be valid timestamps.");
}

@Override
protected Logger getLogger() {
return LOG;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package org.hypertrace.gateway.service.baseline;

import org.hypertrace.gateway.service.common.RequestContext;
import org.hypertrace.gateway.service.v1.baseline.BaselineTimeAggregation;

import java.util.HashMap;
import java.util.Map;

public class BaselineRequestContext extends RequestContext {

private final Map<String, BaselineTimeAggregation> aliasToTimeAggregation = new HashMap();

public BaselineRequestContext(String tenantId, Map<String, String> headers) {
super(tenantId, headers);
}

public void mapAliasToTimeAggregation(String alias, BaselineTimeAggregation timeAggregation) {
aliasToTimeAggregation.put(alias, timeAggregation);
}

public BaselineTimeAggregation getTimeAggregationByAlias(String alias) {
return aliasToTimeAggregation.get(alias);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package org.hypertrace.gateway.service.baseline;

import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesRequest;
import org.hypertrace.gateway.service.v1.baseline.BaselineEntitiesResponse;

import java.util.Map;

public interface BaselineService {
BaselineEntitiesResponse getBaselineForEntities(
String tenantId, BaselineEntitiesRequest originalRequest, Map<String, String> requestHeaders);
}
Loading

0 comments on commit b61884d

Please sign in to comment.