test(ingest): use pytest-random-order for unit tests (#9753)

datahub-project · Feb 9, 2024 · 24da7a5 · 24da7a5
1 parent 93a7afb
commit 24da7a5
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 17 deletions.
diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -142,7 +142,7 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso
   outputs.dir("${venv_name}")
   def cvg_arg = get_coverage_arg("quick")
   commandLine 'bash', '-c',
-    "source ${venv_name}/bin/activate && pytest ${cvg_arg} tests/unit --durations=20 -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "source ${venv_name}/bin/activate && pytest ${cvg_arg} tests/unit --random-order --durations=20 -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
 task installDevTest(type: Exec, dependsOn: [install]) {

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -470,6 +470,7 @@
     "pytest-asyncio>=0.16.0",
     "pytest-cov>=2.8.1",
     "pytest-docker>=1.1.0",
+    "pytest-random-order~=1.1.0",
     deepdiff_dep,
     "requests-mock",
     "freezegun",

diff --git a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py
@@ -416,19 +416,19 @@ def _extract_record(
         # TODO Add topic properties (Pulsar 2.10.0 feature)
         # 3. Construct and emit dataset properties aspect
         if schema is not None:
+            # Add some static properties to the schema properties
             schema_properties = {
+                **schema.properties,
                 "schema_version": str(schema.schema_version),
                 "schema_type": schema.schema_type,
                 "partitioned": str(partitioned).lower(),
             }
-            # Add some static properties to the schema properties
-            schema.properties.update(schema_properties)
 
             yield MetadataChangeProposalWrapper(
                 entityUrn=dataset_urn,
                 aspect=DatasetPropertiesClass(
                     description=schema.schema_description,
-                    customProperties=schema.properties,
+                    customProperties=schema_properties,
                 ),
             ).as_workunit()
 

diff --git a/metadata-ingestion/tests/unit/test_pipeline.py b/metadata-ingestion/tests/unit/test_pipeline.py
@@ -1,5 +1,5 @@
 import pathlib
-from typing import Any, Iterable, List, Optional, cast
+from typing import Iterable, List, cast
 from unittest.mock import patch
 
 import pytest
@@ -24,6 +24,10 @@
 
 FROZEN_TIME = "2020-04-14 07:00:00"
 
+# TODO: It seems like one of these tests writes to ~/.datahubenv or otherwise sets
+# some global config, which impacts other tests.
+pytestmark = pytest.mark.random_order(disabled=True)
+
 
 class TestPipeline(object):
     @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True)
@@ -59,22 +63,16 @@ def test_configure_without_sink(self, mock_emitter, mock_graph):
             {
                 "source": {
                     "type": "file",
-                    "config": {"filename": "test_file.json"},
+                    "config": {"path": "test_file.json"},
                 },
             }
         )
         # assert that the default sink config is for a DatahubRestSink
         assert isinstance(pipeline.config.sink, DynamicTypedConfig)
         assert pipeline.config.sink.type == "datahub-rest"
-        assert pipeline.config.sink.config == {
-            "server": "http://localhost:8080",
-            "token": Optional[Any],
-            # value is read from ~/datahubenv which may be None or not
-        } or pipeline.config.sink.config == {
-            "server": "http://localhost:8080",
-            "token": None,
-            # value is read from ~/datahubenv which may be None or not
-        }
+        assert isinstance(pipeline.config.sink.config, dict)
+        assert pipeline.config.sink.config["server"] == "http://localhost:8080"
+        # token value is read from ~/.datahubenv which may be None or not
 
     @freeze_time(FROZEN_TIME)
     @patch(
@@ -92,7 +90,7 @@ def test_configure_with_rest_sink_initializes_graph(
             {
                 "source": {
                     "type": "file",
-                    "config": {"filename": "test_events.json"},
+                    "config": {"path": "test_events.json"},
                 },
                 "sink": {
                     "type": "datahub-rest",
@@ -130,7 +128,7 @@ def test_configure_with_rest_sink_with_additional_props_initializes_graph(
             {
                 "source": {
                     "type": "file",
-                    "config": {"filename": "test_events.json"},
+                    "config": {"path": "test_events.json"},
                 },
                 "sink": {
                     "type": "datahub-rest",