diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 09c8aae4886d..0dc21e8534df 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -781,7 +781,6 @@ public Map searchLineageInternal( Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); List sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE); - sourceFieldsToExcludeCopy.add("lineage"); searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new)); searchSourceBuilder.query( QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); @@ -1057,7 +1056,6 @@ private void getLineage( List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { Map fromEntity = (HashMap) lin.get("fromEntity"); @@ -1247,7 +1245,6 @@ private Map searchPipelineLineage( List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { HashMap fromEntity = (HashMap) lin.get("fromEntity"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index b8f3a64c9cc4..cc08d612e58c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -777,7 +777,6 @@ public Map searchLineageInternal( Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); List sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE); - sourceFieldsToExcludeCopy.add("lineage"); searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new)); searchSourceBuilder.query( QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn))); @@ -1054,7 +1053,6 @@ private void getLineage( List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { HashMap fromEntity = (HashMap) lin.get("fromEntity"); @@ -1228,7 +1226,6 @@ private Map searchPipelineLineage( List> lineage = (List>) hit.getSourceAsMap().get("lineage"); HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); - tempMap.remove("lineage"); nodes.add(tempMap); for (Map lin : lineage) { HashMap fromEntity = (HashMap) lin.get("fromEntity"); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java index dfb9a671ca41..2e5af9e3ba1b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java @@ -31,11 +31,14 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.BeforeAll; @@ -569,6 +572,42 @@ void get_dataQualityLineage(TestInfo test) deleteEdge(TABLES.get(6), TABLES.get(7)); } + @Order(7) + @Test + void get_SearchLineage(TestInfo testInfo) throws HttpResponseException { + // our lineage is + // 0 + // +-----+-----+ + // v v + // 2 4 + // +---+---+ v + // v | 5 + // 1 | v + // | 6 + // | v + // +-----> 7 + + addEdge(TABLES.get(4), TABLES.get(5)); + addEdge(TABLES.get(5), TABLES.get(6)); + addEdge(TABLES.get(0), TABLES.get(4)); + addEdge(TABLES.get(0), TABLES.get(2)); + addEdge(TABLES.get(2), TABLES.get(1)); + addEdge(TABLES.get(2), TABLES.get(7)); + addEdge(TABLES.get(6), TABLES.get(7)); + + Map>> entity = + searchLineage(TABLES.get(5).getEntityReference(), 1, 1); + assertSearchLineageResponseFields(entity); + + deleteEdge(TABLES.get(4), TABLES.get(5)); + deleteEdge(TABLES.get(5), TABLES.get(6)); + deleteEdge(TABLES.get(0), TABLES.get(4)); + deleteEdge(TABLES.get(0), TABLES.get(2)); + deleteEdge(TABLES.get(2), TABLES.get(1)); + deleteEdge(TABLES.get(2), TABLES.get(7)); + deleteEdge(TABLES.get(6), TABLES.get(7)); + } + public Edge getEdge(Table from, Table to) { return getEdge(from.getId(), to.getId(), null); } @@ -738,6 +777,32 @@ public void assertLineage( assertEquals(lineageById, lineageByName); } + private void assertSearchLineageResponseFields(Map>> entity) { + List> entities = entity.get("nodes"); + Set nodesFields = Set.of("id", "name", "displayName", "fullyQualifiedName", "lineage"); + Set nodesColumnsFields = Set.of("name", "fullyQualifiedName"); + entities.forEach( + e -> { + Set keys = e.keySet(); + Set missingKeys = new HashSet<>(nodesFields); + missingKeys.removeAll(keys); + String err = String.format("Nodes keys not found in the response: %s", missingKeys); + assertTrue(keys.containsAll(nodesFields), err); + + List> columns = (List>) e.get("columns"); + columns.forEach( + c -> { + Set columnsKeys = c.keySet(); + Set missingColumnKeys = new HashSet<>(nodesColumnsFields); + missingColumnKeys.removeAll(columnsKeys); + String columnErr = + String.format( + "Column nodes keys not found in the response: %s", missingColumnKeys); + assertTrue(columnsKeys.containsAll(nodesColumnsFields), columnErr); + }); + }); + } + public EntityLineage getLineage( String entity, UUID id, @@ -754,6 +819,21 @@ public EntityLineage getLineage( return lineage; } + public Map>> searchLineage( + @NonNull EntityReference entityReference, + @NonNull int upstreamDepth, + @NonNull int downstreamDepth) + throws HttpResponseException { + WebTarget target = getResource("lineage/getLineage"); + target = target.queryParam("fqn", entityReference.getFullyQualifiedName()); + target = target.queryParam("type", entityReference.getType()); + target = target.queryParam("upstreamDepth", upstreamDepth); + target = target.queryParam("downstreamDepth", downstreamDepth); + Map>> entity = + TestUtils.get(target, Map.class, ADMIN_AUTH_HEADERS); + return entity; + } + public EntityLineage getLineageByName( String entity, String fqn, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts index b2c6698a7ed4..24f29ec4746d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts @@ -37,6 +37,7 @@ import { deleteNode, editLineage, fillLineageConfigForm, + performExpand, performZoomOut, removeColumnLineage, setupEntitiesForLineage, @@ -374,57 +375,88 @@ test('Verify global lineage config', async ({ browser }) => { const topic = new TopicClass(); const dashboard = new DashboardClass(); const mlModel = new MlModelClass(); + const searchIndex = new SearchIndexClass(); try { await table.create(apiContext); await topic.create(apiContext); await dashboard.create(apiContext); await mlModel.create(apiContext); + await searchIndex.create(apiContext); await addPipelineBetweenNodes(page, table, topic); await addPipelineBetweenNodes(page, topic, dashboard); await addPipelineBetweenNodes(page, dashboard, mlModel); - - await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG); - await fillLineageConfigForm(page, { - upstreamDepth: 1, - downstreamDepth: 1, - layer: 'Column Level Lineage', - }); - - await topic.visitEntityPage(page); - await visitLineageTab(page); - - await verifyNodePresent(page, table); - await verifyNodePresent(page, dashboard); - - const mlModelFqn = get(mlModel, 'entityResponseData.fullyQualifiedName'); - const mlModelNode = page.locator( - `[data-testid="lineage-node-${mlModelFqn}"]` + await addPipelineBetweenNodes(page, mlModel, searchIndex); + + await test.step( + 'Update global lineage config and verify lineage', + async () => { + await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG); + await fillLineageConfigForm(page, { + upstreamDepth: 1, + downstreamDepth: 1, + layer: 'Column Level Lineage', + }); + + await topic.visitEntityPage(page); + await visitLineageTab(page); + await verifyNodePresent(page, table); + await verifyNodePresent(page, dashboard); + const mlModelFqn = get( + mlModel, + 'entityResponseData.fullyQualifiedName' + ); + const mlModelNode = page.locator( + `[data-testid="lineage-node-${mlModelFqn}"]` + ); + + await expect(mlModelNode).not.toBeVisible(); + + await verifyColumnLayerActive(page); + } ); - await expect(mlModelNode).not.toBeVisible(); - - await verifyColumnLayerActive(page); - - await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG); - await fillLineageConfigForm(page, { - upstreamDepth: 2, - downstreamDepth: 2, - layer: 'Entity Lineage', - }); - - await topic.visitEntityPage(page); - await visitLineageTab(page); + await test.step( + 'Verify Upstream and Downstream expand collapse buttons', + async () => { + await dashboard.visitEntityPage(page); + await visitLineageTab(page); + await page.getByTestId('entity-panel-close-icon').click(); + await performZoomOut(page); + await verifyNodePresent(page, topic); + await verifyNodePresent(page, mlModel); + await performExpand(page, mlModel, false, searchIndex); + await performExpand(page, topic, true); + } + ); - await verifyNodePresent(page, table); - await verifyNodePresent(page, dashboard); - await verifyNodePresent(page, mlModel); + await test.step( + 'Reset global lineage config and verify lineage', + async () => { + await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG); + await fillLineageConfigForm(page, { + upstreamDepth: 2, + downstreamDepth: 2, + layer: 'Entity Lineage', + }); + + await dashboard.visitEntityPage(page); + await visitLineageTab(page); + + await verifyNodePresent(page, table); + await verifyNodePresent(page, dashboard); + await verifyNodePresent(page, mlModel); + await verifyNodePresent(page, searchIndex); + await verifyNodePresent(page, topic); + } + ); } finally { await table.delete(apiContext); await topic.delete(apiContext); await dashboard.delete(apiContext); await mlModel.delete(apiContext); + await searchIndex.delete(apiContext); await afterAction(); } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts index cd1f45cc9597..369d0aeda633 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts @@ -161,6 +161,29 @@ export const connectEdgeBetweenNodes = async ( ); }; +export const performExpand = async ( + page: Page, + node: EntityClass, + upstream: boolean, + newNode?: EntityClass +) => { + const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName'); + const handleDirection = upstream ? 'left' : 'right'; + const expandBtn = page + .locator(`[data-testid="lineage-node-${nodeFqn}"]`) + .locator(`.react-flow__handle-${handleDirection}`) + .getByTestId('plus-icon'); + + if (newNode) { + const expandRes = page.waitForResponse('/api/v1/lineage/getLineage?*'); + await expandBtn.click(); + await expandRes; + await verifyNodePresent(page, newNode); + } else { + await expect(expandBtn).toBeVisible(); + } +}; + export const verifyNodePresent = async (page: Page, node: EntityClass) => { const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName'); const name = get(node, 'entityResponseData.name'); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx index 3a45bc5e62ba..fb12ce6c1431 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx @@ -78,7 +78,9 @@ export const getExpandHandle = ( ? 'react-flow__handle-right' : 'react-flow__handle-left' )} - icon={} + icon={ + + } shape="circle" size="small" onClick={(e) => { @@ -106,7 +108,9 @@ export const getCollapseHandle = ( ? 'downstream-collapse-handle' : 'upstream-collapse-handle' } - icon={} + icon={ + + } shape="circle" size="small" onClick={(e) => {