feat(ingest/glossary): support pydantic v2 in business-glossary source

datahub-project · Feb 1, 2024 · ec9b41e · ec9b41e
1 parent 5331304
commit ec9b41e
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 31 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py
@@ -34,44 +34,44 @@
 
 
 class Owners(ConfigModel):
-    users: Optional[List[str]]
-    groups: Optional[List[str]]
+    users: Optional[List[str]] = None
+    groups: Optional[List[str]] = None
 
 
 class KnowledgeCard(ConfigModel):
-    url: Optional[str]
-    label: Optional[str]
+    url: Optional[str] = None
+    label: Optional[str] = None
 
 
 class GlossaryTermConfig(ConfigModel):
-    id: Optional[str]
+    id: Optional[str] = None
     name: str
     description: str
-    term_source: Optional[str]
-    source_ref: Optional[str]
-    source_url: Optional[str]
-    owners: Optional[Owners]
-    inherits: Optional[List[str]]
-    contains: Optional[List[str]]
-    values: Optional[List[str]]
-    related_terms: Optional[List[str]]
-    custom_properties: Optional[Dict[str, str]]
-    knowledge_links: Optional[List[KnowledgeCard]]
-    domain: Optional[str]
+    term_source: Optional[str] = None
+    source_ref: Optional[str] = None
+    source_url: Optional[str] = None
+    owners: Optional[Owners] = None
+    inherits: Optional[List[str]] = None
+    contains: Optional[List[str]] = None
+    values: Optional[List[str]] = None
+    related_terms: Optional[List[str]] = None
+    custom_properties: Optional[Dict[str, str]] = None
+    knowledge_links: Optional[List[KnowledgeCard]] = None
+    domain: Optional[str] = None
 
     # Private fields.
     _urn: str
 
 
 class GlossaryNodeConfig(ConfigModel):
-    id: Optional[str]
+    id: Optional[str] = None
     name: str
     description: str
-    owners: Optional[Owners]
-    terms: Optional[List["GlossaryTermConfig"]]
-    nodes: Optional[List["GlossaryNodeConfig"]]
-    knowledge_links: Optional[List[KnowledgeCard]]
-    custom_properties: Optional[Dict[str, str]]
+    owners: Optional[Owners] = None
+    terms: Optional[List["GlossaryTermConfig"]] = None
+    nodes: Optional[List["GlossaryNodeConfig"]] = None
+    knowledge_links: Optional[List[KnowledgeCard]] = None
+    custom_properties: Optional[Dict[str, str]] = None
 
     # Private fields.
     _urn: str
@@ -80,7 +80,7 @@ class GlossaryNodeConfig(ConfigModel):
 class DefaultConfig(ConfigModel):
     """Holds defaults for populating fields in glossary terms"""
 
-    source: Optional[str]
+    source: Optional[str] = None
     owners: Owners
     url: Optional[str] = None
     source_type: str = "INTERNAL"
@@ -98,8 +98,8 @@ class BusinessGlossarySourceConfig(ConfigModel):
 
 class BusinessGlossaryConfig(DefaultConfig):
     version: str
-    terms: Optional[List["GlossaryTermConfig"]]
-    nodes: Optional[List["GlossaryNodeConfig"]]
+    terms: Optional[List["GlossaryTermConfig"]] = None
+    nodes: Optional[List["GlossaryNodeConfig"]] = None
 
     @validator("version")
     def version_must_be_1(cls, v):
@@ -337,12 +337,14 @@ def get_mces_from_term(
     ] = []
     term_info = models.GlossaryTermInfoClass(
         definition=glossaryTerm.description,
-        termSource=glossaryTerm.term_source
-        if glossaryTerm.term_source is not None
-        else defaults.source_type,
-        sourceRef=glossaryTerm.source_ref
-        if glossaryTerm.source_ref
-        else defaults.source,
+        termSource=(
+            glossaryTerm.term_source
+            if glossaryTerm.term_source is not None
+            else defaults.source_type
+        ),
+        sourceRef=(
+            glossaryTerm.source_ref if glossaryTerm.source_ref else defaults.source
+        ),
         sourceUrl=glossaryTerm.source_url if glossaryTerm.source_url else defaults.url,
         parentNode=parentNode,
         customProperties=glossaryTerm.custom_properties,

diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py
@@ -195,6 +195,7 @@ def get_workspaces(self) -> List[Workspace]:
             groups = self._get_resolver().get_groups()
         except:
             self.log_http_error(message="Unable to fetch list of workspaces")
+            raise  # we want this exception to bubble up
 
         workspaces = [
             Workspace(