From f2e461eb633c0bdd6196f94320ca09cb06fd360b Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Fri, 2 Aug 2024 21:01:46 +0530 Subject: [PATCH] feat(ingest/mode): add option to exclude restricted (#11081) --- .../src/datahub/ingestion/source/mode.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py index 4b4822bcb98ca..3da7f98e93008 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mode.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py @@ -135,9 +135,14 @@ class ModeConfig(StatefulIngestionConfigBase, DatasetLineageProviderConfigBase): connect_uri: str = Field( default="https://app.mode.com", description="Mode host URL." ) - token: str = Field(description="Mode user token.") + token: str = Field( + description="When creating workspace API key this is the 'Key ID'." + ) password: pydantic.SecretStr = Field( - description="Mode password for authentication." + description="When creating workspace API key this is the 'Secret'." + ) + exclude_restricted: bool = Field( + default=False, description="Exclude restricted collections" ) workspace: str = Field( @@ -522,6 +527,16 @@ def _get_space_name_and_tokens(self) -> dict: for s in spaces: logger.debug(f"Space: {s.get('name')}") space_name = s.get("name", "") + # Using both restricted and default_access_level because + # there is a current bug with restricted returning False everytime + # which has been reported to Mode team + if self.config.exclude_restricted and ( + s.get("restricted") or s.get("default_access_level") == "restricted" + ): + logging.debug( + f"Skipping space {space_name} due to exclude restricted" + ) + continue if not self.config.space_pattern.allowed(space_name): self.report.report_dropped_space(space_name) logging.debug(f"Skipping space {space_name} due to space pattern")