increase default sample size, use fraction based sampling

metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py

@@ -162,7 +162,7 @@ class GEProfilingConfig(ConfigModel):
     )
 
     sample_size: int = Field(
-        default=1000,
+        default=10000,
         description="Number of rows to be sampled from table for column level profiling."
         "Applicable only if `use_sampling` is set to True.",
     )

metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_profiler.py

@@ -81,7 +81,12 @@ def get_batch_kwargs(
             and table.rows_count
             and table.rows_count > self.config.profiling.sample_size
         ):
-            custom_sql = f'select * from "{db_name}"."{schema_name}"."{table.name}" TABLESAMPLE ({self.config.profiling.sample_size} rows)'
+            # GX creates a temporary table from query if query is passed as batch kwargs.
+            # We are using fraction-based sampling here, instead of fixed-size sampling because
+            # Fixed-size sampling can be slower than equivalent fraction-based sampling
+            # as per https://docs.snowflake.com/en/sql-reference/constructs/sample#performance-considerations
+            sample_pc = 100 * self.config.profiling.sample_size / table.rows_count
+            custom_sql = f'select * from "{db_name}"."{schema_name}"."{table.name}" TABLESAMPLE ({sample_pc:.3f})'
         return {
             **super().get_batch_kwargs(table, schema_name, db_name),
             # Lowercase/Mixedcase table names in Snowflake do not work by default.