Skip to content
This repository has been archived by the owner on Jun 26, 2024. It is now read-only.

Commit

Permalink
fix(entities): selections on eds, and filters/order by on qs (#70)
Browse files Browse the repository at this point in the history 
* fix(entities): selections on eds, and filters/order by on qs

* fix tests

* added unit tests
  • Loading branch information
@skjindal93
skjindal93 authored Jan 19, 2021
1 parent d50c174 commit 8f44749
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 18 deletions.
34 changes: 21 additions & 13 deletions ...-impl/src/main/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,14 @@ public QueryNode build() {
LOG.debug("Filter Tree:{}", filterTree.acceptVisitor(new PrintVisitor()));
}

// if the filters and order by are on QS, then the DataFetcherNode(s) would have been created
// for QS, with limit and offset pushed down to QS
// But, due to a Pinot limitation of not supporting offset when group by is specified, we need
// to paginate the data in memory by adding a paginate only node
if (sourceSetsIfFilterAndOrderByAreFromSameSourceSets.contains(QS.name())) {
filterTree = createPaginateOnlyNode(filterTree, entitiesRequest);
}

filterTree.acceptVisitor(new ExecutionContextBuilderVisitor(executionContext));

if (LOG.isDebugEnabled()) {
Expand Down Expand Up @@ -159,7 +167,8 @@ private QueryNode buildExecutionTreeForSameSourceFilterAndSelection(String sourc

private QueryNode buildExecutionTreeForQsFilterAndSelection() {
EntitiesRequest entitiesRequest = executionContext.getEntitiesRequest();
QueryNode rootNode = createQsDataFetcherNodeWithPagination(entitiesRequest);
QueryNode rootNode = createQsDataFetcherNodeWithLimitAndOffset(entitiesRequest);
rootNode = createPaginateOnlyNode(rootNode, entitiesRequest);
executionContext.setSortAndPaginationNodeAdded(true);

return rootNode;
Expand Down Expand Up @@ -275,7 +284,7 @@ QueryNode buildFilterTree(EntitiesRequest entitiesRequest, Filter filter) {
// sources, since we will always have a time range filter on QS
if (sourceSetsIfFilterAndOrderByAreFromSameSourceSets.contains(QS.name())) {
executionContext.setSortAndPaginationNodeAdded(true);
return createQsDataFetcherNodeWithPagination(entitiesRequest);
return createQsDataFetcherNodeWithLimitAndOffset(entitiesRequest);
}

return new DataFetcherNode(sources.contains(QS) ? QS.name() : sources.get(0).name(), filter);
Expand Down Expand Up @@ -313,7 +322,7 @@ private QueryNode checkAndAddSortAndPaginationNode(
orderByExpressions);
}

private QueryNode createQsDataFetcherNodeWithPagination(EntitiesRequest entitiesRequest) {
private QueryNode createQsDataFetcherNodeWithLimitAndOffset(EntitiesRequest entitiesRequest) {
Filter filter = entitiesRequest.getFilter();
int selectionLimit = entitiesRequest.getLimit();
int selectionOffset = entitiesRequest.getOffset();
Expand All @@ -329,17 +338,16 @@ private QueryNode createQsDataFetcherNodeWithPagination(EntitiesRequest entities
selectionOffset = 0;
}

QueryNode rootNode =
new DataFetcherNode(QS.name(), filter, selectionLimit, selectionOffset, orderBys);
return new DataFetcherNode(QS.name(), filter, selectionLimit, selectionOffset, orderBys);
}

if (executionContext.getEntitiesRequest().getOffset() > 0) {
rootNode =
new PaginateOnlyNode(
rootNode,
executionContext.getEntitiesRequest().getLimit(),
executionContext.getEntitiesRequest().getOffset());
private QueryNode createPaginateOnlyNode(QueryNode queryNode, EntitiesRequest entitiesRequest) {
if (entitiesRequest.getOffset() > 0) {
return new PaginateOnlyNode(
queryNode,
entitiesRequest.getLimit(),
entitiesRequest.getOffset());
}

return rootNode;
return queryNode;
}
}
18 changes: 15 additions & 3 deletions ...ain/java/org/hypertrace/gateway/service/entity/query/visitor/FilterOptimizingVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,22 @@ private Map<String, List<QueryNode>> groupNodesBySource(List<QueryNode> nodes) {

@Override
public QueryNode visit(SelectionNode selectionNode) {
return selectionNode;
QueryNode childNode = selectionNode.getChildNode().acceptVisitor(this);
return new SelectionNode.Builder(childNode)
.setTimeSeriesSelectionSources(selectionNode.getTimeSeriesSelectionSources())
.setAggMetricSelectionSources(selectionNode.getAggMetricSelectionSources())
.setAttrSelectionSources(selectionNode.getAttrSelectionSources())
.build();
}

@Override
public QueryNode visit(SortAndPaginateNode sortAndPaginateNode) {
return sortAndPaginateNode;
QueryNode childNode = sortAndPaginateNode.getChildNode().acceptVisitor(this);
return new SortAndPaginateNode(
childNode,
sortAndPaginateNode.getLimit(),
sortAndPaginateNode.getOffset(),
sortAndPaginateNode.getOrderByExpressionList());
}

private Map<String, QueryNode> mergeQueryNodesBySource(
Expand Down Expand Up @@ -179,6 +189,8 @@ public QueryNode visit(NoOpNode noOpNode) {

@Override
public QueryNode visit(PaginateOnlyNode paginateOnlyNode) {
return paginateOnlyNode;
QueryNode childNode = paginateOnlyNode.getChildNode().acceptVisitor(this);
return new PaginateOnlyNode(
childNode, paginateOnlyNode.getLimit(), paginateOnlyNode.getOffset());
}
}
55 changes: 55 additions & 0 deletions ...l/src/test/java/org/hypertrace/gateway/service/entity/query/ExecutionTreeBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,61 @@ public void test_build_selectAttributesAndFilterWithSameSourceNonZeroOffset_shou
assertEquals(20, ((DataFetcherNode)dataFetcherNode).getLimit());
}

@Test
public void test_build_selectAttributesAndFilterWithDifferentSourceNonZeroOffset_shouldCreateDataFetcherNodeAndPaginateOnlyNode() {
// selections on EDS and QS
// filters and order by on QS
OrderByExpression orderByExpression = buildOrderByExpression(API_STATE_ATTR);
EntitiesRequest entitiesRequest =
EntitiesRequest.newBuilder()
.setEntityType(AttributeScope.API.name())
.addSelection(buildExpression(API_TYPE_ATTR))
.addSelection(buildExpression(API_STATE_ATTR))
.addSelection(
buildAggregateExpression(API_NUM_CALLS_ATTR,
FunctionType.SUM,
"SUM_numCalls",
List.of()))
.setFilter(generateAndOrNotFilter(
Operator.AND,
generateEQFilter(API_DISCOVERY_STATE, "DISCOVERED"),
generateFilter(Operator.GE, API_NUM_CALLS_ATTR,
Value.newBuilder().
setDouble(60)
.setValueType(ValueType.DOUBLE)
.build()
)
))
.addOrderBy(orderByExpression)
.setLimit(10)
.setOffset(10)
.build();
EntitiesRequestContext entitiesRequestContext =
new EntitiesRequestContext(TENANT_ID, 0L, 10L, "API", "API.startTime", new HashMap<>());
ExecutionContext executionContext =
ExecutionContext.from(attributeMetadataProvider, entityIdColumnsConfigs, entitiesRequest, entitiesRequestContext);
ExecutionTreeBuilder executionTreeBuilder = new ExecutionTreeBuilder(executionContext);
QueryNode executionTree = executionTreeBuilder.build();
assertNotNull(executionTree);
assertTrue(executionTree instanceof SelectionNode);
assertTrue(((SelectionNode) executionTree).getAggMetricSelectionSources().contains("QS"));

QueryNode selectionNode = ((SelectionNode) executionTree).getChildNode();
assertTrue(selectionNode instanceof SelectionNode);
assertTrue(((SelectionNode) selectionNode).getAttrSelectionSources().contains("EDS"));

QueryNode paginateOnlyNode = ((SelectionNode)selectionNode).getChildNode();
assertTrue(paginateOnlyNode instanceof PaginateOnlyNode);
assertEquals(10, ((PaginateOnlyNode)paginateOnlyNode).getOffset());
assertEquals(10, ((PaginateOnlyNode)paginateOnlyNode).getLimit());

QueryNode dataFetcherNode = ((PaginateOnlyNode)paginateOnlyNode).getChildNode();
assertTrue(dataFetcherNode instanceof DataFetcherNode);
assertEquals("QS", ((DataFetcherNode)dataFetcherNode).getSource());
assertEquals(0, ((DataFetcherNode)dataFetcherNode).getOffset());
assertEquals(20, ((DataFetcherNode)dataFetcherNode).getLimit());
}

@Test
public void test_build_selectAttributeAndAggregateMetricWithDifferentSource_shouldCreateDifferentNode() {
EntitiesRequest entitiesRequest =
Expand Down
54 changes: 52 additions & 2 deletions ...java/org/hypertrace/gateway/service/entity/query/visitor/FilterOptimizingVisitorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,70 @@
import org.hypertrace.gateway.service.entity.query.DataFetcherNode;
import org.hypertrace.gateway.service.entity.query.PaginateOnlyNode;
import org.hypertrace.gateway.service.entity.query.QueryNode;
import org.hypertrace.gateway.service.entity.query.SelectionNode;
import org.hypertrace.gateway.service.entity.query.SortAndPaginateNode;
import org.hypertrace.gateway.service.v1.common.Filter;
import org.hypertrace.gateway.service.v1.common.Operator;
import org.hypertrace.gateway.service.v1.common.OrderByExpression;
import org.junit.jupiter.api.Test;

import java.util.Collections;
import java.util.List;
import java.util.Set;

public class FilterOptimizingVisitorTest {
@Test
public void testPaginateOnlyNode() {
PaginateOnlyNode paginateOnlyNode = mock(PaginateOnlyNode.class);
DataFetcherNode dataFetcherNode = new DataFetcherNode("QS", Filter.getDefaultInstance());
PaginateOnlyNode paginateOnlyNode = new PaginateOnlyNode(dataFetcherNode, 10, 10);
FilterOptimizingVisitor filterOptimizingVisitor = new FilterOptimizingVisitor();
assertEquals(paginateOnlyNode, filterOptimizingVisitor.visit(paginateOnlyNode));
PaginateOnlyNode visitedPaginatedOnlyNode = (PaginateOnlyNode) filterOptimizingVisitor.visit(paginateOnlyNode);
assertEquals(paginateOnlyNode.getChildNode(), visitedPaginatedOnlyNode.getChildNode());
assertEquals(paginateOnlyNode.getLimit(), visitedPaginatedOnlyNode.getLimit());
assertEquals(paginateOnlyNode.getOffset(), visitedPaginatedOnlyNode.getOffset());
}

@Test
public void testSelectionNode() {
DataFetcherNode dataFetcherNode = new DataFetcherNode("QS", Filter.getDefaultInstance());
Set<String> attrSelectionSources = Set.of("QS", "EDS");
Set<String> aggregationSources = Set.of("QS");
Set<String> timeAggregationSources = Set.of("source1", "source2");
SelectionNode selectionNode =
new SelectionNode.Builder(dataFetcherNode)
.setAttrSelectionSources(attrSelectionSources)
.setAggMetricSelectionSources(aggregationSources)
.setTimeSeriesSelectionSources(timeAggregationSources)
.build();
FilterOptimizingVisitor filterOptimizingVisitor = new FilterOptimizingVisitor();
SelectionNode visitedSelectionNode =
(SelectionNode) filterOptimizingVisitor.visit(selectionNode);
assertEquals(selectionNode.getChildNode(), visitedSelectionNode.getChildNode());
assertEquals(
selectionNode.getAttrSelectionSources(), visitedSelectionNode.getAttrSelectionSources());
assertEquals(
selectionNode.getAggMetricSelectionSources(),
visitedSelectionNode.getAggMetricSelectionSources());
assertEquals(
selectionNode.getTimeSeriesSelectionSources(),
visitedSelectionNode.getTimeSeriesSelectionSources());
}

@Test
public void testSortAndPaginateNode() {
OrderByExpression orderByExpression = buildOrderByExpression("api1");

DataFetcherNode dataFetcherNode = new DataFetcherNode("QS", Filter.getDefaultInstance());
SortAndPaginateNode sortAndPaginateNode =
new SortAndPaginateNode(dataFetcherNode, 10, 20, List.of(orderByExpression));

FilterOptimizingVisitor filterOptimizingVisitor = new FilterOptimizingVisitor();
SortAndPaginateNode visitedSortAndPaginateNode =
(SortAndPaginateNode) filterOptimizingVisitor.visit(sortAndPaginateNode);
assertEquals(sortAndPaginateNode.getChildNode(), visitedSortAndPaginateNode.getChildNode());
assertEquals(10, visitedSortAndPaginateNode.getLimit());
assertEquals(20, visitedSortAndPaginateNode.getOffset());
assertEquals(List.of(orderByExpression), visitedSortAndPaginateNode.getOrderByExpressionList());
}

@Test
Expand Down

0 comments on commit 8f44749

Please sign in to comment.