Skip to content

Commit

Permalink
option to load schema metadata from datahub
Browse files Browse the repository at this point in the history
  • Loading branch information
asikowitz committed Oct 20, 2023
1 parent 6845885 commit f193ab6
Showing 1 changed file with 19 additions and 6 deletions.
25 changes: 19 additions & 6 deletions metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,12 +251,7 @@ def __init__(self, config: TeradataConfig, ctx: PipelineContext):
generate_operations=self.config.usage.include_operational_stats,
)

self.schema_resolver = SchemaResolver(
platform=self.platform,
platform_instance=self.config.platform_instance,
graph=None,
env=self.config.env,
)
self.schema_resolver = self._init_schema_resolver()

if self.config.use_file_backed_cache:
self._view_definition_cache = FileBackedDict[str]()
Expand Down Expand Up @@ -301,6 +296,24 @@ def create(cls, config_dict, ctx):
config = TeradataConfig.parse_obj(config_dict)
return cls(config, ctx)

def _init_schema_resolver(self) -> SchemaResolver:
if self.config.disable_schema_metadata:
if self.ctx.graph:
return self.ctx.graph.initialize_schema_resolver_from_datahub(
platform=self.platform,
platform_instance=self.config.platform_instance,
env=self.config.env,
)
else:
logger.warning(
"Failed to load schema info from DataHub as DataHubGraph is missing.",
)
return SchemaResolver(
platform=self.platform,
platform_instance=self.config.platform_instance,
env=self.config.env,
)

def get_inspectors(self):
# This method can be overridden in the case that you want to dynamically
# run on multiple databases.
Expand Down

0 comments on commit f193ab6

Please sign in to comment.