Adding more log to Redshift usage

datahub-project · Nov 21, 2023 · c13458a · c13458a
1 parent 1ad4f96
commit c13458a
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py
@@ -31,6 +31,7 @@ class RedshiftReport(ProfilingSqlReport, IngestionStageReport, BaseTimeWindowRep
     views_in_mem_size: Dict[str, str] = field(default_factory=TopKDict)
     num_operational_stats_skipped: int = 0
     num_usage_stat_skipped: int = 0
+    num_usage_stat_processed: int = 0
     num_lineage_tables_dropped: int = 0
     num_lineage_dropped_query_parser: int = 0
     num_lineage_dropped_not_support_copy_path: int = 0

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/usage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/usage.py
@@ -333,6 +333,7 @@ def _gen_access_events_from_history_query(
         results = cursor.fetchmany()
         field_names = [i[0] for i in cursor.description]
         while results:
+            logger.debug(f"Processing {len(results)} rows of access events")
             for row in results:
                 try:
                     access_event = RedshiftAccessEvent(
@@ -368,6 +369,8 @@ def _gen_access_events_from_history_query(
                     continue
 
                 yield access_event
+                self.report.num_usage_stat_processed += 1
+
             results = cursor.fetchmany()
 
     def _gen_operation_aspect_workunits_from_access_events(