Skip to content

Commit

Permalink
refactor(rust): Use size_of/align_of from prelude (#19311)
Browse files Browse the repository at this point in the history
  • Loading branch information
orlp authored Oct 19, 2024
1 parent da8e37a commit a871f8c
Show file tree
Hide file tree
Showing 61 changed files with 134 additions and 153 deletions.
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/array/primitive/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,8 @@ impl<T: NativeType> PrimitiveArray<T> {

// SAFETY: this is fine, we checked size and alignment, and NativeType
// is always Pod.
assert_eq!(std::mem::size_of::<T>(), std::mem::size_of::<U>());
assert_eq!(std::mem::align_of::<T>(), std::mem::align_of::<U>());
assert_eq!(size_of::<T>(), size_of::<U>());
assert_eq!(align_of::<T>(), align_of::<U>());
let new_values = unsafe { std::mem::transmute::<Buffer<T>, Buffer<U>>(values) };
PrimitiveArray::new(U::PRIMITIVE.into(), new_values, validity)
}
Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/aligned.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ impl<'a, T: BitChunk> AlignedBitmapSlice<'a, T> {
/// The length (in bits) of the portion of the bitmap found in bulk.
#[inline(always)]
pub fn bulk_bitlen(&self) -> usize {
8 * std::mem::size_of::<T>() * self.bulk.len()
8 * size_of::<T>() * self.bulk.len()
}

/// The length (in bits) of the portion of the bitmap found in suffix.
Expand All @@ -77,7 +77,7 @@ impl<'a, T: BitChunk> AlignedBitmapSlice<'a, T> {
offset %= 8;

// Fast-path: fits entirely in one chunk.
let chunk_len = std::mem::size_of::<T>();
let chunk_len = size_of::<T>();
let chunk_len_bits = 8 * chunk_len;
if offset + len <= chunk_len_bits {
let mut prefix = load_chunk_le::<T>(bytes) >> offset;
Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/bitmap_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ pub(crate) fn push_bitchunk<T: BitChunk>(buffer: &mut Vec<u8>, value: T) {

/// Creates a [`Vec<u8>`] from a [`TrustedLen`] of [`BitChunk`].
pub fn chunk_iter_to_vec<T: BitChunk, I: TrustedLen<Item = T>>(iter: I) -> Vec<u8> {
let cap = iter.size_hint().0 * std::mem::size_of::<T>();
let cap = iter.size_hint().0 * size_of::<T>();
let mut buffer = Vec::with_capacity(cap);
for v in iter {
push_bitchunk(&mut buffer, v)
Expand All @@ -24,7 +24,7 @@ fn chunk_iter_to_vec_and_remainder<T: BitChunk, I: TrustedLen<Item = T>>(
iter: I,
remainder: T,
) -> Vec<u8> {
let cap = (iter.size_hint().0 + 1) * std::mem::size_of::<T>();
let cap = (iter.size_hint().0 + 1) * size_of::<T>();
let mut buffer = Vec::with_capacity(cap);
for v in iter {
push_bitchunk(&mut buffer, v)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ impl<'a, T: BitChunk> BitChunksExact<'a, T> {
#[inline]
pub fn new(bitmap: &'a [u8], length: usize) -> Self {
assert!(length <= bitmap.len() * 8);
let size_of = std::mem::size_of::<T>();
let size_of = size_of::<T>();

let bitmap = &bitmap[..length.saturating_add(7) / 8];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ where
// expected = [n5, n6, n7, c0, c1, c2, c3, c4]

// 1. unset most significants of `next` up to `offset`
let inverse_offset = std::mem::size_of::<T>() * 8 - offset;
let inverse_offset = size_of::<T>() * 8 - offset;
next <<= inverse_offset;
// next = [n5, n6, n7, 0 , 0 , 0 , 0 , 0 ]

Expand Down
8 changes: 4 additions & 4 deletions crates/polars-arrow/src/bitmap/utils/chunk_iterator/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ fn copy_with_merge<T: BitChunk>(dst: &mut T::Bytes, bytes: &[u8], bit_offset: us
bytes
.windows(2)
.chain(std::iter::once([bytes[bytes.len() - 1], 0].as_ref()))
.take(std::mem::size_of::<T>())
.take(size_of::<T>())
.enumerate()
.for_each(|(i, w)| {
let val = merge_reversed(w[0], w[1], bit_offset);
Expand All @@ -59,7 +59,7 @@ impl<'a, T: BitChunk> BitChunks<'a, T> {

let slice = &slice[offset / 8..];
let bit_offset = offset % 8;
let size_of = std::mem::size_of::<T>();
let size_of = size_of::<T>();

let bytes_len = len / 8;
let bytes_upper_len = (len + bit_offset + 7) / 8;
Expand Down Expand Up @@ -120,7 +120,7 @@ impl<'a, T: BitChunk> BitChunks<'a, T> {
// all remaining bytes
self.remainder_bytes
.iter()
.take(std::mem::size_of::<T>())
.take(size_of::<T>())
.enumerate()
.for_each(|(i, val)| remainder[i] = *val);

Expand All @@ -137,7 +137,7 @@ impl<'a, T: BitChunk> BitChunks<'a, T> {

/// Returns the remainder bits in [`BitChunks::remainder`].
pub fn remainder_len(&self) -> usize {
self.len - (std::mem::size_of::<T>() * ((self.len / 8) / std::mem::size_of::<T>()) * 8)
self.len - (size_of::<T>() * ((self.len / 8) / size_of::<T>()) * 8)
}
}

Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/utils/chunks_exact_mut.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use super::BitChunk;
///
/// # Safety
/// The slices returned by this iterator are guaranteed to have length equal to
/// `std::mem::size_of::<T>()`.
/// `size_of::<T>()`.
#[derive(Debug)]
pub struct BitChunksExactMut<'a, T: BitChunk> {
chunks: std::slice::ChunksExactMut<'a, u8>,
Expand All @@ -18,7 +18,7 @@ impl<'a, T: BitChunk> BitChunksExactMut<'a, T> {
#[inline]
pub fn new(bitmap: &'a mut [u8], length: usize) -> Self {
assert!(length <= bitmap.len() * 8);
let size_of = std::mem::size_of::<T>();
let size_of = size_of::<T>();

let bitmap = &mut bitmap[..length.saturating_add(7) / 8];

Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/bitmap/utils/iterator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ impl<'a> BitmapIter<'a> {
let num_words = n / 64;

if num_words > 0 {
assert!(self.bytes.len() >= num_words * std::mem::size_of::<u64>());
assert!(self.bytes.len() >= num_words * size_of::<u64>());

bitmap.extend_from_slice(self.bytes, 0, num_words * u64::BITS as usize);

Expand All @@ -189,7 +189,7 @@ impl<'a> BitmapIter<'a> {
return;
}

assert!(self.bytes.len() >= std::mem::size_of::<u64>());
assert!(self.bytes.len() >= size_of::<u64>());

self.word_len = usize::min(self.rest_len, 64);
self.rest_len -= self.word_len;
Expand Down
8 changes: 4 additions & 4 deletions crates/polars-arrow/src/buffer/immutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -299,10 +299,10 @@ impl<T: crate::types::NativeType> From<arrow_buffer::Buffer> for Buffer<T> {
impl<T: crate::types::NativeType> From<Buffer<T>> for arrow_buffer::Buffer {
fn from(value: Buffer<T>) -> Self {
let offset = value.offset();
value.storage.into_arrow_buffer().slice_with_length(
offset * std::mem::size_of::<T>(),
value.length * std::mem::size_of::<T>(),
)
value
.storage
.into_arrow_buffer()
.slice_with_length(offset * size_of::<T>(), value.length * size_of::<T>())
}
}

Expand Down
16 changes: 8 additions & 8 deletions crates/polars-arrow/src/compute/aggregate/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ macro_rules! dyn_binary {
let values_end = offsets[offsets.len() - 1] as usize;

values_end - values_start
+ offsets.len() * std::mem::size_of::<$o>()
+ offsets.len() * size_of::<$o>()
+ validity_size(array.validity())
}};
}
Expand Down Expand Up @@ -50,15 +50,15 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
},
Primitive(PrimitiveType::DaysMs) => {
let array = array.as_any().downcast_ref::<DaysMsArray>().unwrap();
array.values().len() * std::mem::size_of::<i32>() * 2 + validity_size(array.validity())
array.values().len() * size_of::<i32>() * 2 + validity_size(array.validity())
},
Primitive(primitive) => with_match_primitive_type_full!(primitive, |$T| {
let array = array
.as_any()
.downcast_ref::<PrimitiveArray<$T>>()
.unwrap();

array.values().len() * std::mem::size_of::<$T>() + validity_size(array.validity())
array.values().len() * size_of::<$T>() + validity_size(array.validity())
}),
Binary => dyn_binary!(array, BinaryArray<i32>, i32),
FixedSizeBinary => {
Expand All @@ -74,7 +74,7 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
List => {
let array = array.as_any().downcast_ref::<ListArray<i32>>().unwrap();
estimated_bytes_size(array.values().as_ref())
+ array.offsets().len_proxy() * std::mem::size_of::<i32>()
+ array.offsets().len_proxy() * size_of::<i32>()
+ validity_size(array.validity())
},
FixedSizeList => {
Expand All @@ -84,7 +84,7 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
LargeList => {
let array = array.as_any().downcast_ref::<ListArray<i64>>().unwrap();
estimated_bytes_size(array.values().as_ref())
+ array.offsets().len_proxy() * std::mem::size_of::<i64>()
+ array.offsets().len_proxy() * size_of::<i64>()
+ validity_size(array.validity())
},
Struct => {
Expand All @@ -99,11 +99,11 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
},
Union => {
let array = array.as_any().downcast_ref::<UnionArray>().unwrap();
let types = array.types().len() * std::mem::size_of::<i8>();
let types = array.types().len() * size_of::<i8>();
let offsets = array
.offsets()
.as_ref()
.map(|x| x.len() * std::mem::size_of::<i32>())
.map(|x| x.len() * size_of::<i32>())
.unwrap_or_default();
let fields = array
.fields()
Expand All @@ -124,7 +124,7 @@ pub fn estimated_bytes_size(array: &dyn Array) -> usize {
BinaryView => binview_size::<[u8]>(array.as_any().downcast_ref().unwrap()),
Map => {
let array = array.as_any().downcast_ref::<MapArray>().unwrap();
let offsets = array.offsets().len_proxy() * std::mem::size_of::<i32>();
let offsets = array.offsets().len_proxy() * size_of::<i32>();
offsets + estimated_bytes_size(array.field().as_ref()) + validity_size(array.validity())
},
}
Expand Down
8 changes: 2 additions & 6 deletions crates/polars-arrow/src/ffi/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,11 +225,7 @@ unsafe fn get_buffer_ptr<T: NativeType>(
);
}

if array
.buffers
.align_offset(std::mem::align_of::<*mut *const u8>())
!= 0
{
if array.buffers.align_offset(align_of::<*mut *const u8>()) != 0 {
polars_bail!( ComputeError:
"an ArrowArray of type {dtype:?}
must have buffer {index} aligned to type {}",
Expand Down Expand Up @@ -294,7 +290,7 @@ unsafe fn create_buffer<T: NativeType>(

// We have to check alignment.
// This is the zero-copy path.
if ptr.align_offset(std::mem::align_of::<T>()) == 0 {
if ptr.align_offset(align_of::<T>()) == 0 {
let storage = SharedStorage::from_internal_arrow_array(ptr, len, owner);
Ok(Buffer::from_storage(storage).sliced(offset, len - offset))
}
Expand Down
14 changes: 6 additions & 8 deletions crates/polars-arrow/src/io/avro/read/deserialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -195,19 +195,17 @@ fn deserialize_value<'a>(
array.push(Some(value))
},
PrimitiveType::Float32 => {
let value =
f32::from_le_bytes(block[..std::mem::size_of::<f32>()].try_into().unwrap());
block = &block[std::mem::size_of::<f32>()..];
let value = f32::from_le_bytes(block[..size_of::<f32>()].try_into().unwrap());
block = &block[size_of::<f32>()..];
let array = array
.as_mut_any()
.downcast_mut::<MutablePrimitiveArray<f32>>()
.unwrap();
array.push(Some(value))
},
PrimitiveType::Float64 => {
let value =
f64::from_le_bytes(block[..std::mem::size_of::<f64>()].try_into().unwrap());
block = &block[std::mem::size_of::<f64>()..];
let value = f64::from_le_bytes(block[..size_of::<f64>()].try_into().unwrap());
block = &block[size_of::<f64>()..];
let array = array
.as_mut_any()
.downcast_mut::<MutablePrimitiveArray<f64>>()
Expand Down Expand Up @@ -404,10 +402,10 @@ fn skip_item<'a>(
let _ = util::zigzag_i64(&mut block)?;
},
PrimitiveType::Float32 => {
block = &block[std::mem::size_of::<f32>()..];
block = &block[size_of::<f32>()..];
},
PrimitiveType::Float64 => {
block = &block[std::mem::size_of::<f64>()..];
block = &block[size_of::<f64>()..];
},
PrimitiveType::MonthDayNano => {
block = &block[12..];
Expand Down
6 changes: 3 additions & 3 deletions crates/polars-arrow/src/io/ipc/read/read_basic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ fn read_swapped<T: NativeType, R: Read + Seek>(
is_little_endian: bool,
) -> PolarsResult<()> {
// slow case where we must reverse bits
let mut slice = vec![0u8; length * std::mem::size_of::<T>()];
let mut slice = vec![0u8; length * size_of::<T>()];
reader.read_exact(&mut slice)?;

let chunks = slice.chunks_exact(std::mem::size_of::<T>());
let chunks = slice.chunks_exact(size_of::<T>());
if !is_little_endian {
// machine is little endian, file is big endian
buffer
Expand Down Expand Up @@ -67,7 +67,7 @@ fn read_uncompressed_buffer<T: NativeType, R: Read + Seek>(
length: usize,
is_little_endian: bool,
) -> PolarsResult<Vec<T>> {
let required_number_of_bytes = length.saturating_mul(std::mem::size_of::<T>());
let required_number_of_bytes = length.saturating_mul(size_of::<T>());
if required_number_of_bytes > buffer_length {
polars_bail!(
oos = OutOfSpecKind::InvalidBuffer {
Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/io/ipc/write/serialize/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ fn _write_buffer_from_iter<T: NativeType, I: TrustedLen<Item = T>>(
is_little_endian: bool,
) {
let len = buffer.size_hint().0;
arrow_data.reserve(len * std::mem::size_of::<T>());
arrow_data.reserve(len * size_of::<T>());
if is_little_endian {
buffer
.map(|x| T::to_le_bytes(&x))
Expand All @@ -303,7 +303,7 @@ fn _write_compressed_buffer_from_iter<T: NativeType, I: TrustedLen<Item = T>>(
compression: Compression,
) {
let len = buffer.size_hint().0;
let mut swapped = Vec::with_capacity(len * std::mem::size_of::<T>());
let mut swapped = Vec::with_capacity(len * size_of::<T>());
if is_little_endian {
buffer
.map(|x| T::to_le_bytes(&x))
Expand Down
3 changes: 1 addition & 2 deletions crates/polars-arrow/src/legacy/kernels/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,7 @@ impl<'a> MaskedSlicesIterator<'a> {
pub(crate) fn new(mask: &'a BooleanArray) -> Self {
let chunks = mask.values().chunks::<u64>();

let chunk_bits = 8 * std::mem::size_of::<u64>();
let chunk_len = mask.len() / chunk_bits;
let chunk_len = mask.len() / 64;
let remainder_len = chunks.remainder_len();
let remainder_mask = chunks.remainder();

Expand Down
4 changes: 2 additions & 2 deletions crates/polars-arrow/src/mmap/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ fn check_bytes_len_and_is_aligned<T: NativeType>(
bytes: &[u8],
expected_len: usize,
) -> PolarsResult<bool> {
if bytes.len() < std::mem::size_of::<T>() * expected_len {
if bytes.len() < size_of::<T>() * expected_len {
polars_bail!(ComputeError: "buffer's length is too small in mmap")
};

Expand Down Expand Up @@ -281,7 +281,7 @@ fn mmap_primitive<P: NativeType, T: AsRef<[u8]>>(
let bytes = get_bytes(data_ref, block_offset, buffers)?;
let is_aligned = check_bytes_len_and_is_aligned::<P>(bytes, num_rows)?;

let out = if is_aligned || std::mem::size_of::<T>() <= 8 {
let out = if is_aligned || size_of::<T>() <= 8 {
assert!(
is_aligned,
"primitive type with size <= 8 bytes should have been aligned"
Expand Down
6 changes: 3 additions & 3 deletions crates/polars-arrow/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,9 @@ impl<T> SharedStorage<T> {
impl<T: crate::types::NativeType> SharedStorage<T> {
pub fn from_arrow_buffer(buffer: arrow_buffer::Buffer) -> Self {
let ptr = buffer.as_ptr();
let align_offset = ptr.align_offset(std::mem::align_of::<T>());
let align_offset = ptr.align_offset(align_of::<T>());
assert_eq!(align_offset, 0, "arrow_buffer::Buffer misaligned");
let length = buffer.len() / std::mem::size_of::<T>();
let length = buffer.len() / size_of::<T>();

let inner = SharedStorageInner {
ref_count: AtomicU64::new(1),
Expand All @@ -119,7 +119,7 @@ impl<T: crate::types::NativeType> SharedStorage<T> {

pub fn into_arrow_buffer(self) -> arrow_buffer::Buffer {
let ptr = NonNull::new(self.as_ptr() as *mut u8).unwrap();
let len = self.len() * std::mem::size_of::<T>();
let len = self.len() * size_of::<T>();
let arc = std::sync::Arc::new(self);
unsafe { arrow_buffer::Buffer::from_custom_allocation(ptr, len, arc) }
}
Expand Down
2 changes: 1 addition & 1 deletion crates/polars-arrow/src/types/bit_chunk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ impl<T: BitChunk> BitChunkIter<T> {
/// Creates a new [`BitChunkIter`] with `len` bits.
#[inline]
pub fn new(value: T, len: usize) -> Self {
assert!(len <= std::mem::size_of::<T>() * 8);
assert!(len <= size_of::<T>() * 8);
Self {
value,
remaining: len,
Expand Down
2 changes: 1 addition & 1 deletion crates/polars-arrow/src/types/native.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ macro_rules! native_type {
impl NativeType for $type {
const PRIMITIVE: PrimitiveType = $primitive_type;

type Bytes = [u8; std::mem::size_of::<Self>()];
type Bytes = [u8; size_of::<Self>()];
#[inline]
fn to_le_bytes(&self) -> Self::Bytes {
Self::to_le_bytes(*self)
Expand Down
Loading

0 comments on commit a871f8c

Please sign in to comment.