diff --git a/crates/polars-core/src/config.rs b/crates/polars-core/src/config.rs index 7b9b0b878009..572db05a677e 100644 --- a/crates/polars-core/src/config.rs +++ b/crates/polars-core/src/config.rs @@ -28,10 +28,3 @@ pub(crate) fn decimal_is_active() -> bool { pub fn verbose() -> bool { std::env::var("POLARS_VERBOSE").as_deref().unwrap_or("") == "1" } - -pub fn concurrent_download_limit() -> usize { - std::env::var("POLARS_CONC_DOWNLOAD_LIMIT") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(64) -} diff --git a/crates/polars-lazy/src/physical_plan/executors/scan/parquet.rs b/crates/polars-lazy/src/physical_plan/executors/scan/parquet.rs index 03be23b5cd51..287187d5cad0 100644 --- a/crates/polars-lazy/src/physical_plan/executors/scan/parquet.rs +++ b/crates/polars-lazy/src/physical_plan/executors/scan/parquet.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use polars_core::config::{concurrent_download_limit, verbose}; +use polars_core::config::verbose; use polars_core::utils::accumulate_dataframes_vertical; use polars_io::cloud::CloudOptions; use polars_io::parquet::FileMetaData; @@ -165,25 +165,7 @@ impl ParquetExec { let cloud_options = self.cloud_options.as_ref(); let mut result = vec![]; - let batch_size = if let Some(md) = self.metadata.as_ref() { - let n_columns = self - .file_options - .with_columns - .as_ref() - .map(|opt| opt.len()) - .unwrap_or(first_schema.len()); - let concurrent_per_file = md.row_groups.len() * n_columns; - if verbose { - eprintln!( - "estimated concurrent downloads per file: {}", - concurrent_per_file - ); - } - concurrent_download_limit() / concurrent_per_file + 1 - } else { - std::cmp::min(POOL.current_num_threads(), 16) - }; - + let batch_size = 5; let mut remaining_rows_to_read = self.file_options.n_rows.unwrap_or(usize::MAX); let mut base_row_count = self.file_options.row_count.take(); let mut processed = 0;