From f6825c2ac140a8ef5e5cadc169e28a724c417406 Mon Sep 17 00:00:00 2001 From: jaidisido Date: Mon, 28 Oct 2024 14:02:29 +0000 Subject: [PATCH] fix: remove _default_metadata_providers (#3007) --- .../distributed/ray/datasources/arrow_parquet_datasource.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/awswrangler/distributed/ray/datasources/arrow_parquet_datasource.py b/awswrangler/distributed/ray/datasources/arrow_parquet_datasource.py index 919fcdcf4..041423645 100644 --- a/awswrangler/distributed/ray/datasources/arrow_parquet_datasource.py +++ b/awswrangler/distributed/ray/datasources/arrow_parquet_datasource.py @@ -30,11 +30,9 @@ from ray.data.block import Block from ray.data.context import DataContext from ray.data.datasource import Datasource -from ray.data.datasource._default_metadata_providers import ( - get_generic_metadata_provider, -) from ray.data.datasource.datasource import ReadTask from ray.data.datasource.file_meta_provider import ( + DefaultFileMetadataProvider, _handle_read_os_error, ) from ray.data.datasource.parquet_meta_provider import ( @@ -219,7 +217,7 @@ def __init__( # noqa: PLR0912,PLR0915 # files. To avoid this, we expand the input paths with the default metadata # provider and then apply the partition filter or file extensions. if partition_filter is not None or file_extensions is not None: - default_meta_provider = get_generic_metadata_provider(file_extensions=None) + default_meta_provider = DefaultFileMetadataProvider() expanded_paths, _ = map(list, zip(*default_meta_provider.expand_paths(paths, filesystem))) paths = list(expanded_paths)