Merge branch 'master' into update-pydantic-pin

asikowitz · Aug 23, 2023 · 1ebca9c · 1ebca9c
2 parents d9e2f4c + d6e36f1
commit 1ebca9c
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 28 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -549,6 +549,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
+      needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
     needs: [setup, datahub_ingestion_base_slim_build]
     steps:
       - name: Check out the repo
@@ -605,7 +606,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Download image Slim Image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Slim Image
@@ -630,6 +631,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
+      needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.publish != 'true' }}
     needs: [setup, datahub_ingestion_base_full_build]
     steps:
       - name: Check out the repo
@@ -685,7 +687,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Download image Full Image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.datahub_ingestion_full_build.outputs.needs_artifact_download == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
       - name: Run Trivy vulnerability scanner Full Image
@@ -792,7 +794,7 @@ jobs:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
       - name: Download datahub-ingestion-slim image
         uses: ishworkh/docker-image-artifact-download@v1
-        if: ${{ needs.setup.outputs.publish != 'true' }}
+        if: ${{ needs.datahub_ingestion_slim_build.outputs.needs_artifact_download == 'true' }}
         with:
           image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
       - name: Disk Check

diff --git a/metadata-ingestion/tests/integration/vertica/docker-compose.yml b/metadata-ingestion/tests/integration/vertica/docker-compose.yml
@@ -1,6 +1,7 @@
 version: "3.9"
 services:
   vertica:
+    platform: linux/amd64
     environment:
       APP_DB_USER: "dbadmin"
       APP_DB_PASSWORD: "abc123"
@@ -18,6 +19,3 @@ services:
 
 volumes:
   vertica-data:
-
-
-
diff --git a/metadata-ingestion/tests/integration/vertica/test_vertica.py b/metadata-ingestion/tests/integration/vertica/test_vertica.py
@@ -58,6 +58,7 @@ def vertica_runner(docker_compose_runner, test_resources_dir):
 
 # Test needs more work to be done , currently it is working fine.
 @freeze_time(FROZEN_TIME)
+@pytest.mark.skip("Failing in CI, cmd failing with exit code 1")
 @pytest.mark.integration
 def test_vertica_ingest_with_db(vertica_runner, pytestconfig, tmp_path):
     test_resources_dir = pytestconfig.rootpath / "tests/integration/vertica"

diff --git a/...est/java/com/linkedin/metadata/search/elasticsearch/fixtures/ElasticSearchGoldenTest.java b/...est/java/com/linkedin/metadata/search/elasticsearch/fixtures/ElasticSearchGoldenTest.java
@@ -15,7 +15,6 @@
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Import;
 import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
-import org.testng.annotations.Ignore;
 import org.testng.annotations.Test;
 
 import java.util.List;
@@ -80,24 +79,6 @@ public void testNameMatchPetProfile() {
     }
 
     @Test
-    public void testNameMatchMemberInWorkspace() {
-        /*
-          Searching for "collaborative actionitems" should return "collaborative_actionitems" as the first search
-          result, followed by "collaborative_actionitems_old"
-         */
-        assertNotNull(searchService);
-        SearchResult searchResult = searchAcrossEntities(searchService, "collaborative actionitems", SEARCHABLE_LONGTAIL_ENTITIES);
-        assertTrue(searchResult.getEntities().size() >= 2);
-        Urn firstResultUrn = searchResult.getEntities().get(0).getEntity();
-        Urn secondResultUrn = searchResult.getEntities().get(1).getEntity();
-
-        // Checks that the table name is not suffixed with anything
-        assertTrue(firstResultUrn.toString().contains("collaborative_actionitems,"));
-        assertTrue(secondResultUrn.toString().contains("collaborative_actionitems_old"));
-    }
-
-    @Test
-    @Ignore("unstable")
     public void testGlossaryTerms() {
         /*
           Searching for "ReturnRate" should return all tables that have the glossary term applied before
@@ -134,9 +115,53 @@ public void testNameMatchPartiallyQualified() {
         assertTrue(secondResultUrn.toString().contains("dbt,long_tail_companions.analytics.pet_details"));
     }
 
+    @Test
+    public void testNameMatchCollaborativeActionitems() {
+        /*
+          Searching for "collaborative actionitems" should return "collaborative_actionitems" as the first search
+          result, followed by "collaborative_actionitems_old"
+         */
+        assertNotNull(searchService);
+        SearchResult searchResult = searchAcrossEntities(searchService, "collaborative actionitems", SEARCHABLE_LONGTAIL_ENTITIES);
+        assertTrue(searchResult.getEntities().size() >= 2);
+        Urn firstResultUrn = searchResult.getEntities().get(0).getEntity();
+        Urn secondResultUrn = searchResult.getEntities().get(1).getEntity();
+
+        // Checks that the table name is not suffixed with anything
+        assertTrue(firstResultUrn.toString().contains("collaborative_actionitems,"));
+        assertTrue(secondResultUrn.toString().contains("collaborative_actionitems_old"));
+
+        Double firstResultScore = searchResult.getEntities().get(0).getScore();
+        Double secondResultScore = searchResult.getEntities().get(1).getScore();
+
+        // Checks that the scores aren't tied so that we are matching on table name more than column name
+        assertTrue(firstResultScore > secondResultScore);
+    }
+
+    @Test
+    public void testNameMatchCustomerOrders() {
+        /*
+          Searching for "customer orders" should return "customer_orders" as the first search
+          result, not suffixed by anything
+         */
+        assertNotNull(searchService);
+        SearchResult searchResult = searchAcrossEntities(searchService, "customer orders", SEARCHABLE_LONGTAIL_ENTITIES);
+        assertTrue(searchResult.getEntities().size() >= 2);
+        Urn firstResultUrn = searchResult.getEntities().get(0).getEntity();
+
+        // Checks that the table name is not suffixed with anything
+        assertTrue(firstResultUrn.toString().contains("customer_orders,"));
+
+        Double firstResultScore = searchResult.getEntities().get(0).getScore();
+        Double secondResultScore = searchResult.getEntities().get(1).getScore();
+
+        // Checks that the scores aren't tied so that we are matching on table name more than column name
+        assertTrue(firstResultScore > secondResultScore);
+    }
+
     /*
-     * Tests that should pass but do not yet can be added below here, with the following annotation:
-     * @Test(enabled = false)
-     **/
+      Tests that should pass but do not yet can be added below here, with the following annotation:
+      @Test(enabled = false)
+     */
 
 }