Skip to content
This repository has been archived by the owner on Jun 26, 2024. It is now read-only.

Commit

Permalink
Merge remote-tracking branch 'origin/main' into aman/eds_filter
Browse files Browse the repository at this point in the history
  • Loading branch information
aman-bansal committed Nov 29, 2023
2 parents 2d0b2d0 + 9f2aa9d commit 2de7d25
Show file tree
Hide file tree
Showing 19 changed files with 283 additions and 44 deletions.
1 change: 1 addition & 0 deletions .trivyignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CVE-2023-5678 exp:2023-11-30
16 changes: 8 additions & 8 deletions build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
plugins {
id("org.hypertrace.repository-plugin") version "0.4.1"
id("org.hypertrace.ci-utils-plugin") version "0.3.0"
id("org.hypertrace.publish-plugin") version "1.0.2" apply false
id("org.hypertrace.jacoco-report-plugin") version "0.2.0" apply false
id("org.hypertrace.docker-java-application-plugin") version "0.9.4" apply false
id("org.hypertrace.docker-publish-plugin") version "0.9.4" apply false
id("org.hypertrace.code-style-plugin") version "1.1.0" apply false
id("org.hypertrace.repository-plugin") version "0.4.2"
id("org.hypertrace.ci-utils-plugin") version "0.3.2"
id("org.hypertrace.publish-plugin") version "1.0.5" apply false
id("org.hypertrace.jacoco-report-plugin") version "0.2.1" apply false
id("org.hypertrace.docker-java-application-plugin") version "0.9.9" apply false
id("org.hypertrace.docker-publish-plugin") version "0.9.9" apply false
id("org.hypertrace.code-style-plugin") version "1.2.0" apply false
id("org.owasp.dependencycheck") version "8.1.2"
}

Expand All @@ -32,4 +32,4 @@ dependencyCheck {
suppressionFile = "owasp-suppressions.xml"
scanConfigurations.add("runtimeClasspath")
failBuildOnCVSS = 7.0F
}
}
6 changes: 3 additions & 3 deletions gateway-service-api/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ protobuf {
// the identifier, which can be referred to in the "plugins"
// container of the "generateProtoTasks" closure.
id("grpc_java") {
artifact = "io.grpc:protoc-gen-grpc-java:1.56.0"
artifact = "io.grpc:protoc-gen-grpc-java:1.57.2"
}
}
generateProtoTasks {
Expand All @@ -44,12 +44,12 @@ sourceSets {
}

dependencies {
api(platform("io.grpc:grpc-bom:1.56.0"))
api(platform("io.grpc:grpc-bom:1.57.2"))
api("io.grpc:grpc-protobuf")
api("io.grpc:grpc-stub")
api("javax.annotation:javax.annotation-api:1.3.2")
constraints {
implementation("com.google.guava:guava:32.0.1-jre") {
implementation("com.google.guava:guava:32.1.2-jre") {
because("https://nvd.nist.gov/vuln/detail/CVE-2023-2976")
}
}
Expand Down
2 changes: 1 addition & 1 deletion gateway-service-factory/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ plugins {
}

dependencies {
api("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.52")
api("org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.62")

implementation(project(":gateway-service-impl"))
}
20 changes: 10 additions & 10 deletions gateway-service-impl/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,28 @@ dependencies {
implementation("org.hypertrace.core.query.service:query-service-client:0.8.0")
implementation("org.hypertrace.core.attribute.service:attribute-service-client:0.14.25")

implementation("org.hypertrace.entity.service:entity-service-client:0.8.56")
implementation("org.hypertrace.entity.service:entity-service-api:0.8.56")
implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.12.0")
implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.12.0")
implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.52")
implementation("org.hypertrace.entity.service:entity-service-client:0.8.87")
implementation("org.hypertrace.entity.service:entity-service-api:0.8.87")
implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.12.5")
implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.12.5")
implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.58")

// Config
implementation("com.typesafe:config:1.4.1")

// Common utilities
implementation("org.apache.commons:commons-lang3:3.12.0")
implementation("com.google.protobuf:protobuf-java-util:3.21.12")
implementation("com.google.guava:guava:32.0.1-jre")
implementation("com.google.guava:guava:32.1.2-jre")
implementation("com.google.inject:guice:5.0.1")

implementation("com.fasterxml.jackson.core:jackson-annotations:2.15.2")
implementation("com.fasterxml.jackson.core:jackson-databind:2.15.2")

testImplementation("org.junit.jupiter:junit-jupiter:5.8.2")
testImplementation("org.mockito:mockito-junit-jupiter:4.10.0")
testImplementation("org.mockito:mockito-core:4.10.0")
testImplementation("org.mockito:mockito-inline:4.10.0")
testImplementation("org.mockito:mockito-junit-jupiter:5.4.0")
testImplementation("org.mockito:mockito-core:5.5.0")
testImplementation("org.mockito:mockito-inline:5.2.0")
testImplementation("org.apache.logging.log4j:log4j-slf4j-impl:2.17.1")
testImplementation("io.grpc:grpc-netty:1.56.0")
testImplementation("io.grpc:grpc-netty:1.57.2")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package org.hypertrace.gateway.service;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import lombok.NonNull;
import org.hypertrace.core.grpcutils.context.ContextualKey;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceClient;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceV2Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class EntityTypesProvider {
private static final Logger LOG = LoggerFactory.getLogger(EntityTypesProvider.class);
private static final int DEFAULT_CACHE_SIZE = 1024;
private static final int DEFAULT_EXPIRE_DURATION_MIN = 30; // 30 minutes

// TenantId to Entity types cache
private final LoadingCache<ContextualKey<Void>, Set<String>> tenantIdToEntityTypesCache;

public EntityTypesProvider(
EntityTypeServiceClient entityTypeServiceClient,
EntityTypeServiceV2Client entityTypeServiceV2Client) {
CacheLoader<ContextualKey<Void>, Set<String>> cacheLoader =
new CacheLoader<>() {
@Override
public Set<String> load(@NonNull ContextualKey<Void> contextualKey) {
Set<String> entityTypesSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
entityTypeServiceClient
.getAllEntityTypes(contextualKey.getContext())
.forEach(entityType -> entityTypesSet.add(entityType.getName()));
entityTypeServiceV2Client
.getAllEntityTypes(contextualKey.getContext())
.forEach(entityType -> entityTypesSet.add(entityType.getName()));
return entityTypesSet;
}
};

tenantIdToEntityTypesCache =
CacheBuilder.newBuilder()
.maximumSize(DEFAULT_CACHE_SIZE)
.expireAfterWrite(DEFAULT_EXPIRE_DURATION_MIN, TimeUnit.MINUTES)
.build(cacheLoader);
}

public Set<String> getEntityTypes(ContextualKey<Void> contextualKey) {
try {
return tenantIdToEntityTypesCache.get(contextualKey);
} catch (ExecutionException e) {
LOG.error(
String.format(
"Error retrieving entity types for tenant %s",
contextualKey.getContext().getTenantId()));
throw new RuntimeException(
String.format(
"Error retrieving Entity types for tenant:%s",
contextualKey.getContext().getTenantId()));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.ServiceException;
import com.google.protobuf.util.JsonFormat;
import com.typesafe.config.Config;
import io.grpc.stub.StreamObserver;
import io.micrometer.core.instrument.Counter;
Expand All @@ -29,6 +30,8 @@
import org.hypertrace.gateway.service.common.AttributeMetadataProvider;
import org.hypertrace.gateway.service.common.RequestContext;
import org.hypertrace.gateway.service.common.config.ScopeFilterConfigs;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceClient;
import org.hypertrace.gateway.service.common.util.EntityTypeServiceV2Client;
import org.hypertrace.gateway.service.common.util.QueryServiceClient;
import org.hypertrace.gateway.service.entity.EntityService;
import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs;
Expand Down Expand Up @@ -111,6 +114,16 @@ public GatewayServiceImpl(Config appConfig, GrpcChannelRegistry grpcChannelRegis
new EntityQueryServiceClient(
grpcChannelRegistry.forPlaintextAddress(esConfig.getHost(), esConfig.getPort()));

EntityTypeServiceClient entityTypeServiceClient =
new EntityTypeServiceClient(
grpcChannelRegistry.forPlaintextAddress(esConfig.getHost(), esConfig.getPort()));

EntityTypeServiceV2Client entityTypeServiceV2Client =
new EntityTypeServiceV2Client(
grpcChannelRegistry.forPlaintextAddress(esConfig.getHost(), esConfig.getPort()));
EntityTypesProvider entityTypesProvider =
new EntityTypesProvider(entityTypeServiceClient, entityTypeServiceV2Client);

ScopeFilterConfigs scopeFilterConfigs = new ScopeFilterConfigs(appConfig);
LogConfig logConfig = new LogConfig(appConfig);
this.traceService =
Expand All @@ -134,7 +147,8 @@ public GatewayServiceImpl(Config appConfig, GrpcChannelRegistry grpcChannelRegis
eqsClient,
attributeMetadataProvider,
scopeFilterConfigs,
entityIdColumnsConfigs);
entityIdColumnsConfigs,
entityTypesProvider);
BaselineServiceQueryParser baselineServiceQueryParser =
new BaselineServiceQueryParser(attributeMetadataProvider);
BaselineServiceQueryExecutor baselineServiceQueryExecutor =
Expand Down Expand Up @@ -404,7 +418,12 @@ public void explore(ExploreRequest request, StreamObserver<ExploreResponse> resp
responseObserver.onCompleted();
requestStatusSuccessCounter.increment();
} catch (Exception e) {
LOG.error("Error while handling explore request: {}", request, e);
try {
LOG.error(
"Error while handling explore request: {}", JsonFormat.printer().print(request), e);
} catch (Exception ex) {
LOG.error("Error while handling explore request: {}", request, e);
}
requestStatusErrorCounter.increment();
responseObserver.onError(e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -364,13 +364,12 @@ private List<EntityInteractionQueryRequest> prepareQueryRequests(
.collect(Collectors.toList());
}

private Set<String> getOtherEntityTypes(org.hypertrace.gateway.service.v1.common.Filter filter) {
protected Set<String> getOtherEntityTypes(
org.hypertrace.gateway.service.v1.common.Filter filter) {
Set<String> result = new HashSet<>();
if (filter.getChildFilterCount() > 0) {
for (org.hypertrace.gateway.service.v1.common.Filter child : filter.getChildFilterList()) {
Set<String> result = getOtherEntityTypes(child);
if (!result.isEmpty()) {
return result;
}
result.addAll(getOtherEntityTypes(child));
}
} else if (ExpressionReader.isSimpleAttributeSelection(filter.getLhs())) {
String attributeId =
Expand All @@ -382,7 +381,7 @@ private Set<String> getOtherEntityTypes(org.hypertrace.gateway.service.v1.common
}
}

return Collections.emptySet();
return Collections.unmodifiableSet(result);
}

private Filter convertToQueryFilter(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package org.hypertrace.gateway.service.common.util;

import com.google.common.collect.Lists;
import io.grpc.Channel;
import java.util.List;
import org.hypertrace.core.grpcutils.client.RequestContextClientCallCredsProviderFactory;
import org.hypertrace.core.grpcutils.context.RequestContext;
import org.hypertrace.entity.type.service.v1.EntityType;
import org.hypertrace.entity.type.service.v1.EntityTypeFilter;
import org.hypertrace.entity.type.service.v1.EntityTypeServiceGrpc;

public class EntityTypeServiceClient {
private final EntityTypeServiceGrpc.EntityTypeServiceBlockingStub blockingStub;

public EntityTypeServiceClient(Channel channel) {
this.blockingStub =
EntityTypeServiceGrpc.newBlockingStub(channel)
.withCallCredentials(
RequestContextClientCallCredsProviderFactory.getClientCallCredsProvider().get());
}

public List<EntityType> getAllEntityTypes(RequestContext requestContext) {
return Lists.newArrayList(
requestContext.call(
() -> this.blockingStub.queryEntityTypes(EntityTypeFilter.newBuilder().build())));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package org.hypertrace.gateway.service.common.util;

import io.grpc.Channel;
import java.util.List;
import org.hypertrace.core.grpcutils.client.RequestContextClientCallCredsProviderFactory;
import org.hypertrace.core.grpcutils.context.RequestContext;
import org.hypertrace.entity.type.service.v2.EntityType;
import org.hypertrace.entity.type.service.v2.EntityTypeServiceGrpc;
import org.hypertrace.entity.type.service.v2.QueryEntityTypesRequest;

public class EntityTypeServiceV2Client {
private final EntityTypeServiceGrpc.EntityTypeServiceBlockingStub blockingStub;

public EntityTypeServiceV2Client(Channel channel) {
this.blockingStub =
EntityTypeServiceGrpc.newBlockingStub(channel)
.withCallCredentials(
RequestContextClientCallCredsProviderFactory.getClientCallCredsProvider().get());
}

public List<EntityType> getAllEntityTypes(RequestContext requestContext) {
return requestContext
.call(
() -> this.blockingStub.queryEntityTypes(QueryEntityTypesRequest.newBuilder().build()))
.getEntityTypeList();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.hypertrace.core.serviceframework.metrics.PlatformMetricsRegistry;
import org.hypertrace.entity.query.service.client.EntityQueryServiceClient;
import org.hypertrace.entity.v1.entitytype.EntityType;
import org.hypertrace.gateway.service.EntityTypesProvider;
import org.hypertrace.gateway.service.common.AttributeMetadataProvider;
import org.hypertrace.gateway.service.common.ExpressionContext;
import org.hypertrace.gateway.service.common.RequestContext;
Expand All @@ -36,6 +37,7 @@ public class ExploreService {
private final TimeAggregationsWithGroupByRequestHandler timeAggregationsWithGroupByRequestHandler;
private final EntityRequestHandler entityRequestHandler;
private final ScopeFilterConfigs scopeFilterConfigs;
private final EntityTypesProvider entityTypesProvider;

private Timer queryExecutionTimer;

Expand All @@ -44,14 +46,14 @@ public ExploreService(
EntityQueryServiceClient entityQueryServiceClient,
AttributeMetadataProvider attributeMetadataProvider,
ScopeFilterConfigs scopeFiltersConfig,
EntityIdColumnsConfigs entityIdColumnsConfigs) {
EntityIdColumnsConfigs entityIdColumnsConfigs,
EntityTypesProvider entityTypesProvider) {
QueryServiceEntityFetcher queryServiceEntityFetcher =
new QueryServiceEntityFetcher(
queryServiceClient, attributeMetadataProvider, entityIdColumnsConfigs);
EntityServiceEntityFetcher entityServiceEntityFetcher =
new EntityServiceEntityFetcher(
attributeMetadataProvider, entityIdColumnsConfigs, entityQueryServiceClient);

this.attributeMetadataProvider = attributeMetadataProvider;
this.normalRequestHandler =
new RequestHandler(
Expand All @@ -78,6 +80,7 @@ public ExploreService(
queryServiceEntityFetcher,
entityServiceEntityFetcher);
this.scopeFilterConfigs = scopeFiltersConfig;
this.entityTypesProvider = entityTypesProvider;
initMetrics();
}

Expand Down Expand Up @@ -112,7 +115,8 @@ public ExploreResponse explore(RequestContext requestContext, ExploreRequest req
newExploreRequestContext, request.getContext());
exploreRequestValidator.validate(request, attributeMetadataMap);

IRequestHandler requestHandler = getRequestHandler(request, attributeMetadataMap);
IRequestHandler requestHandler =
getRequestHandler(request, attributeMetadataMap, requestContext);

ExploreResponse.Builder responseBuilder =
requestHandler.handleRequest(newExploreRequestContext, request);
Expand All @@ -124,14 +128,21 @@ public ExploreResponse explore(RequestContext requestContext, ExploreRequest req
}
}

private boolean isContextAnEntityType(ExploreRequest request) {
private boolean isContextAnEntityType(ExploreRequest request, RequestContext requestContext) {
if (entityTypesProvider
.getEntityTypes(requestContext.getGrpcContext().buildInternalContextualKey())
.contains(request.getContext())) {
return true;
}
return Arrays.stream(EntityType.values())
.anyMatch(entityType -> entityType.name().equalsIgnoreCase(request.getContext()));
}

private IRequestHandler getRequestHandler(
ExploreRequest request, Map<String, AttributeMetadata> attributeMetadataMap) {
if (isContextAnEntityType(request)
ExploreRequest request,
Map<String, AttributeMetadata> attributeMetadataMap,
RequestContext requestContext) {
if (isContextAnEntityType(request, requestContext)
&& !hasTimeAggregations(request)
&& !request.getGroupByList().isEmpty()) {
ExpressionContext expressionContext =
Expand Down
Loading

0 comments on commit 2de7d25

Please sign in to comment.