Skip to content

Commit

Permalink
fix: Pyarrow 15 imports & remove unused code (#2649)
Browse files Browse the repository at this point in the history
Signed-off-by: Anton Kukushkin <[email protected]>
  • Loading branch information
kukushking authored Jan 29, 2024
1 parent 3cd9601 commit 153fe89
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 9 deletions.
9 changes: 2 additions & 7 deletions awswrangler/_data_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -702,17 +702,12 @@ def pyarrow_schema_from_pandas(

def athena_types_from_pyarrow_schema(
schema: pa.Schema,
partitions: pyarrow.parquet.ParquetPartitions | None,
ignore_null: bool = False,
) -> tuple[dict[str, str], dict[str, str] | None]:
) -> dict[str, str]:
"""Extract the related Athena data types from any PyArrow Schema considering possible partitions."""
columns_types: dict[str, str] = {str(f.name): pyarrow2athena(dtype=f.type, ignore_null=ignore_null) for f in schema}
_logger.debug("columns_types: %s", columns_types)
partitions_types: dict[str, str] | None = None
if partitions is not None:
partitions_types = {p.name: pyarrow2athena(p.dictionary.type, ignore_null=ignore_null) for p in partitions}
_logger.debug("partitions_types: %s", partitions_types)
return columns_types, partitions_types
return columns_types


def cast_pandas_with_athena_types(
Expand Down
4 changes: 2 additions & 2 deletions awswrangler/s3/_read.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,8 @@ def read_table_metadata(
merged_schemas = _validate_schemas(schemas=schemas, validate_schema=False)

columns_types: dict[str, str] = _data_types.athena_types_from_pyarrow_schema(
schema=merged_schemas, partitions=None, ignore_null=ignore_null
)[0]
schema=merged_schemas, ignore_null=ignore_null
)

# Partitions
partitions_types: dict[str, str] | None = None
Expand Down

0 comments on commit 153fe89

Please sign in to comment.