From 7c57220199bfd394c1a95df729127977bef4bb3b Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Sep 2023 10:19:38 -0700 Subject: [PATCH 1/5] fix: use `_resolve_element` method in config load --- datahub-actions/setup.py | 7 +++++-- datahub-actions/src/datahub_actions/cli/actions.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/datahub-actions/setup.py b/datahub-actions/setup.py index 6dba39c4..f5d66111 100644 --- a/datahub-actions/setup.py +++ b/datahub-actions/setup.py @@ -29,6 +29,7 @@ def get_long_description(): return description + acryl_datahub_min_version = os.environ.get("ACRYL_DATAHUB_MIN_VERSION", "0.9.4") acryl_datahub_min_version = os.environ.get("ACRYL_DATAHUB_MIN_VERSION") or "0.10.3" @@ -74,7 +75,7 @@ def get_long_description(): "kafka": set(), # included by default # Action Plugins "executor": { - "acryl-executor==0.0.3.11", + "acryl-executor==0.0.3.12", }, "slack": { "slack-bolt>=1.15.5", @@ -84,7 +85,9 @@ def get_long_description(): }, "tag_propagation": set(), "term_propagation": set(), - "snowflake_tag_propagation": {f"acryl-datahub[snowflake]>={acryl_datahub_min_version}"} + "snowflake_tag_propagation": { + f"acryl-datahub[snowflake]>={acryl_datahub_min_version}" + } # Transformer Plugins (None yet) } diff --git a/datahub-actions/src/datahub_actions/cli/actions.py b/datahub-actions/src/datahub_actions/cli/actions.py index b6c1f41b..0f2c3c1f 100644 --- a/datahub-actions/src/datahub_actions/cli/actions.py +++ b/datahub-actions/src/datahub_actions/cli/actions.py @@ -22,7 +22,7 @@ import click from click_default_group import DefaultGroup -from datahub.configuration.config_loader import load_config_file, resolve_element +from datahub.configuration.config_loader import load_config_file, _resolve_element import datahub_actions as datahub_actions_package from datahub_actions.pipeline.pipeline import Pipeline @@ -37,7 +37,7 @@ def best_effort_resolve_element(x: str) -> str: try: - return resolve_element(x) + return _resolve_element(x) except Exception: return x @@ -94,7 +94,7 @@ def run(ctx: Any, config: List[str], debug: bool) -> None: for pipeline_config in config: pipeline_config_file = pathlib.Path(pipeline_config) with unittest.mock.patch( - "datahub.configuration.config_loader.resolve_element" + "datahub.configuration.config_loader._resolve_element" ) as mock_resolve_element: mock_resolve_element.side_effect = best_effort_resolve_element pipeline_config_dict = load_config_file(pipeline_config_file) From 81d31d92c664f9029d43b954dffb3070e0e796da Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Sep 2023 10:24:31 -0700 Subject: [PATCH 2/5] fix lint --- datahub-actions/src/datahub_actions/cli/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-actions/src/datahub_actions/cli/actions.py b/datahub-actions/src/datahub_actions/cli/actions.py index 0f2c3c1f..f56d2e5e 100644 --- a/datahub-actions/src/datahub_actions/cli/actions.py +++ b/datahub-actions/src/datahub_actions/cli/actions.py @@ -22,7 +22,7 @@ import click from click_default_group import DefaultGroup -from datahub.configuration.config_loader import load_config_file, _resolve_element +from datahub.configuration.config_loader import _resolve_element, load_config_file import datahub_actions as datahub_actions_package from datahub_actions.pipeline.pipeline import Pipeline From 77290671b6365fe5831fc39a9b771ab9ece576ab Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Sep 2023 10:39:57 -0700 Subject: [PATCH 3/5] fix lint --- datahub-actions/tests/unit/pipeline/test_pipeline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datahub-actions/tests/unit/pipeline/test_pipeline.py b/datahub-actions/tests/unit/pipeline/test_pipeline.py index d62f1e49..0ed3e413 100644 --- a/datahub-actions/tests/unit/pipeline/test_pipeline.py +++ b/datahub-actions/tests/unit/pipeline/test_pipeline.py @@ -32,13 +32,13 @@ def test_create(): # Validate Pipeline is initialized assert valid_pipeline.name is not None assert valid_pipeline.source is not None - assert type(valid_pipeline.source) == TestEventSource + assert isinstance(valid_pipeline.source, TestEventSource) assert valid_pipeline.transforms is not None assert len(valid_pipeline.transforms) == 2 # Filter + Custom - assert type(valid_pipeline.transforms[0]) == FilterTransformer - assert type(valid_pipeline.transforms[1]) == TestTransformer + assert isinstance(valid_pipeline.transforms[0], FilterTransformer) + assert isinstance(valid_pipeline.transforms[1], TestTransformer) assert valid_pipeline.action is not None - assert type(valid_pipeline.action) == TestAction + assert isinstance(valid_pipeline.action, TestAction) assert valid_pipeline._shutdown is False assert valid_pipeline._stats is not None assert valid_pipeline._retry_count == 3 From d4a3d72cc9b282ad389ea069287641f169b7398a Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Sep 2023 10:44:36 -0700 Subject: [PATCH 4/5] update --- datahub-actions/setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datahub-actions/setup.py b/datahub-actions/setup.py index f5d66111..04e095d0 100644 --- a/datahub-actions/setup.py +++ b/datahub-actions/setup.py @@ -30,9 +30,7 @@ def get_long_description(): return description -acryl_datahub_min_version = os.environ.get("ACRYL_DATAHUB_MIN_VERSION", "0.9.4") - -acryl_datahub_min_version = os.environ.get("ACRYL_DATAHUB_MIN_VERSION") or "0.10.3" +acryl_datahub_min_version = os.environ.get("ACRYL_DATAHUB_MIN_VERSION") or "0.11.0" base_requirements = { f"acryl-datahub[kafka]>={acryl_datahub_min_version}", From f0143ac865c8916f87ebeb2821b205625ed0a704 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Sep 2023 10:46:45 -0700 Subject: [PATCH 5/5] fix other lint issue --- datahub-actions/src/datahub_actions/utils/name_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-actions/src/datahub_actions/utils/name_resolver.py b/datahub-actions/src/datahub_actions/utils/name_resolver.py index f6c76f0b..ba6ff9b5 100644 --- a/datahub-actions/src/datahub_actions/utils/name_resolver.py +++ b/datahub-actions/src/datahub_actions/utils/name_resolver.py @@ -130,7 +130,7 @@ class SchemaFieldNameResolver(DefaultNameResolver): def get_entity_name( self, entity_urn: Urn, datahub_graph: Optional[DataHubGraph] ) -> str: - return DatasetUrn._get_simple_field_path_from_v2_field_path( + return DatasetUrn.get_simple_field_path_from_v2_field_path( entity_urn.get_entity_id()[1] )