diff --git a/crates/polars-arrow/src/legacy/array/mod.rs b/crates/polars-arrow/src/legacy/array/mod.rs index 489cfbb0dbeb..18e5a386df0e 100644 --- a/crates/polars-arrow/src/legacy/array/mod.rs +++ b/crates/polars-arrow/src/legacy/array/mod.rs @@ -185,14 +185,6 @@ pub trait ListFromIter { } impl ListFromIter for ListArray {} -pub trait PolarsArray: Array { - fn has_validity(&self) -> bool { - self.validity().is_some() - } -} - -impl PolarsArray for A {} - fn is_nested_null(data_type: &ArrowDataType) -> bool { match data_type { ArrowDataType::Null => true, diff --git a/crates/polars-core/src/chunked_array/builder/list/mod.rs b/crates/polars-core/src/chunked_array/builder/list/mod.rs index 2c642f7e8fc6..9a7f9243dcf3 100644 --- a/crates/polars-core/src/chunked_array/builder/list/mod.rs +++ b/crates/polars-core/src/chunked_array/builder/list/mod.rs @@ -10,7 +10,6 @@ mod primitive; pub use anonymous::*; use arrow::legacy::array::list::AnonymousBuilder; use arrow::legacy::array::null::MutableNullArray; -use arrow::legacy::prelude::*; pub use binary::*; pub use boolean::*; #[cfg(feature = "dtype-categorical")] diff --git a/crates/polars-core/src/chunked_array/builder/list/primitive.rs b/crates/polars-core/src/chunked_array/builder/list/primitive.rs index 34e12433db7a..d9555716d45d 100644 --- a/crates/polars-core/src/chunked_array/builder/list/primitive.rs +++ b/crates/polars-core/src/chunked_array/builder/list/primitive.rs @@ -119,7 +119,7 @@ where let values = self.builder.mut_values(); ca.downcast_iter().for_each(|arr| { - if !arr.has_validity() { + if arr.null_count() == 0 { values.extend_from_slice(arr.values().as_slice()) } else { // SAFETY: diff --git a/crates/polars-core/src/chunked_array/logical/categorical/mod.rs b/crates/polars-core/src/chunked_array/logical/categorical/mod.rs index 1b2fa0d06a7f..32d8a79691da 100644 --- a/crates/polars-core/src/chunked_array/logical/categorical/mod.rs +++ b/crates/polars-core/src/chunked_array/logical/categorical/mod.rs @@ -341,7 +341,7 @@ impl LogicalType for CategoricalChunked { let f = |idx: u32| mapping.get(idx); - if !self.physical.has_validity() { + if !self.physical.has_nulls() { self.physical .into_no_null_iter() .for_each(|idx| builder.append_value(f(idx))); diff --git a/crates/polars-core/src/chunked_array/mod.rs b/crates/polars-core/src/chunked_array/mod.rs index 4b1cfec05140..fe145f7c8b76 100644 --- a/crates/polars-core/src/chunked_array/mod.rs +++ b/crates/polars-core/src/chunked_array/mod.rs @@ -498,10 +498,9 @@ impl ChunkedArray { } #[inline] - /// Return if any the chunks in this [`ChunkedArray`] have a validity bitmap. - /// no bitmap means no null values. - pub fn has_validity(&self) -> bool { - self.iter_validities().any(|valid| valid.is_some()) + /// Return if any the chunks in this [`ChunkedArray`] have nulls. + pub fn has_nulls(&self) -> bool { + self.null_count > 0 } /// Shrink the capacity of this array to fit its length. diff --git a/crates/polars-core/src/chunked_array/ops/apply.rs b/crates/polars-core/src/chunked_array/ops/apply.rs index 632afe52d889..145c898b0bee 100644 --- a/crates/polars-core/src/chunked_array/ops/apply.rs +++ b/crates/polars-core/src/chunked_array/ops/apply.rs @@ -500,7 +500,7 @@ impl<'a> ChunkApply<'a, Series> for ListChunked { out }; let mut ca: ListChunked = { - if !self.has_validity() { + if !self.has_nulls() { self.into_no_null_iter() .map(&mut function) .collect_trusted() diff --git a/crates/polars-core/src/chunked_array/ops/chunkops.rs b/crates/polars-core/src/chunked_array/ops/chunkops.rs index 8a147e0fe123..d97af95367e6 100644 --- a/crates/polars-core/src/chunked_array/ops/chunkops.rs +++ b/crates/polars-core/src/chunked_array/ops/chunkops.rs @@ -368,7 +368,7 @@ impl ObjectChunked { // todo! use iterators once implemented // no_null path - if !self.has_validity() { + if !self.has_nulls() { for arr in chunks { for idx in 0..arr.len() { builder.append_value(arr.value(idx).clone()) diff --git a/crates/polars-core/src/chunked_array/ops/explode.rs b/crates/polars-core/src/chunked_array/ops/explode.rs index f469209558b6..b44ee0863a98 100644 --- a/crates/polars-core/src/chunked_array/ops/explode.rs +++ b/crates/polars-core/src/chunked_array/ops/explode.rs @@ -69,7 +69,7 @@ where // value and collect the indices. // because the length of the array is not known, we first collect the null indexes, offset // with the insertion of empty rows (as None) and later create a validity bitmap - if arr.has_validity() { + if arr.null_count() > 0 { let validity_values = arr.validity().unwrap(); for &o in &offsets[1..] { diff --git a/crates/polars-core/src/chunked_array/ops/nulls.rs b/crates/polars-core/src/chunked_array/ops/nulls.rs index d8044a890ff6..c0ba435c4a51 100644 --- a/crates/polars-core/src/chunked_array/ops/nulls.rs +++ b/crates/polars-core/src/chunked_array/ops/nulls.rs @@ -6,7 +6,7 @@ use crate::chunked_array::metadata::MetadataProperties; impl ChunkedArray { /// Get a mask of the null values. pub fn is_null(&self) -> BooleanChunked { - if !self.has_validity() { + if !self.has_nulls() { return BooleanChunked::full(self.name(), false, self.len()); } // dispatch to non-generic function diff --git a/crates/polars-core/src/chunked_array/ops/set.rs b/crates/polars-core/src/chunked_array/ops/set.rs index abafab312c6a..77aba5673ea2 100644 --- a/crates/polars-core/src/chunked_array/ops/set.rs +++ b/crates/polars-core/src/chunked_array/ops/set.rs @@ -47,7 +47,7 @@ where idx: I, value: Option, ) -> PolarsResult { - if !self.has_validity() { + if !self.has_nulls() { if let Some(value) = value { // Fast path uses kernel. if self.chunks.len() == 1 { @@ -94,7 +94,7 @@ where check_bounds!(self, mask); // Fast path uses the kernel in polars-arrow. - if let (Some(value), false) = (value, mask.has_validity()) { + if let (Some(value), false) = (value, mask.has_nulls()) { let (left, mask) = align_chunks_binary(self, mask); // Apply binary kernel. diff --git a/crates/polars-core/src/chunked_array/ops/unique/mod.rs b/crates/polars-core/src/chunked_array/ops/unique/mod.rs index 989e30061478..40e16f08ed95 100644 --- a/crates/polars-core/src/chunked_array/ops/unique/mod.rs +++ b/crates/polars-core/src/chunked_array/ops/unique/mod.rs @@ -75,7 +75,7 @@ where macro_rules! arg_unique_ca { ($ca:expr) => {{ - match $ca.has_validity() { + match $ca.has_nulls() { false => arg_unique($ca.into_no_null_iter(), $ca.len()), _ => arg_unique($ca.iter(), $ca.len()), } diff --git a/crates/polars-core/src/frame/group_by/aggregations/mod.rs b/crates/polars-core/src/frame/group_by/aggregations/mod.rs index f7a2e9063625..bd805d4bd332 100644 --- a/crates/polars-core/src/frame/group_by/aggregations/mod.rs +++ b/crates/polars-core/src/frame/group_by/aggregations/mod.rs @@ -951,7 +951,7 @@ where } else if idx.len() == 1 { self.get(first as usize).map(|sum| sum.to_f64().unwrap()) } else { - match (self.has_validity(), self.chunks.len()) { + match (self.has_nulls(), self.chunks.len()) { (false, 1) => { take_agg_no_null_primitive_iter_unchecked::<_, f64, _, _>( arr, diff --git a/crates/polars-core/src/frame/group_by/into_groups.rs b/crates/polars-core/src/frame/group_by/into_groups.rs index d507459e7c49..0ae653389af4 100644 --- a/crates/polars-core/src/frame/group_by/into_groups.rs +++ b/crates/polars-core/src/frame/group_by/into_groups.rs @@ -46,7 +46,7 @@ where .collect::>(); group_by_threaded_iter(&keys, n_partitions, sorted) } - } else if !ca.has_validity() { + } else if !ca.has_nulls() { group_by(ca.into_no_null_iter(), sorted) } else { group_by(ca.iter(), sorted) diff --git a/crates/polars-core/src/frame/mod.rs b/crates/polars-core/src/frame/mod.rs index e1c2c9e148ea..9b7338c9eedb 100644 --- a/crates/polars-core/src/frame/mod.rs +++ b/crates/polars-core/src/frame/mod.rs @@ -1072,7 +1072,7 @@ impl DataFrame { }; // fast path for no nulls in df - if iter.clone().all(|s| !s.has_validity()) { + if iter.clone().all(|s| !s.has_nulls()) { return Ok(self.clone()); } diff --git a/crates/polars-core/src/series/implementations/array.rs b/crates/polars-core/src/series/implementations/array.rs index 14ffa58a0614..bc3ed6d23243 100644 --- a/crates/polars-core/src/series/implementations/array.rs +++ b/crates/polars-core/src/series/implementations/array.rs @@ -163,8 +163,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } fn is_null(&self) -> BooleanChunked { diff --git a/crates/polars-core/src/series/implementations/binary.rs b/crates/polars-core/src/series/implementations/binary.rs index 042e58fcf813..4c92802861a3 100644 --- a/crates/polars-core/src/series/implementations/binary.rs +++ b/crates/polars-core/src/series/implementations/binary.rs @@ -194,8 +194,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/binary_offset.rs b/crates/polars-core/src/series/implementations/binary_offset.rs index b0ac481f682c..e3612e2cbe15 100644 --- a/crates/polars-core/src/series/implementations/binary_offset.rs +++ b/crates/polars-core/src/series/implementations/binary_offset.rs @@ -163,8 +163,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } fn is_null(&self) -> BooleanChunked { diff --git a/crates/polars-core/src/series/implementations/boolean.rs b/crates/polars-core/src/series/implementations/boolean.rs index 9c33538c369d..275e2d1a26a7 100644 --- a/crates/polars-core/src/series/implementations/boolean.rs +++ b/crates/polars-core/src/series/implementations/boolean.rs @@ -222,8 +222,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/categorical.rs b/crates/polars-core/src/series/implementations/categorical.rs index 5204048bda81..5db3344726e7 100644 --- a/crates/polars-core/src/series/implementations/categorical.rs +++ b/crates/polars-core/src/series/implementations/categorical.rs @@ -248,8 +248,8 @@ impl SeriesTrait for SeriesWrap { self.0.physical().null_count() } - fn has_validity(&self) -> bool { - self.0.physical().has_validity() + fn has_nulls(&self) -> bool { + self.0.physical().has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/date.rs b/crates/polars-core/src/series/implementations/date.rs index e0658ceb32ce..44a8ee6b32bf 100644 --- a/crates/polars-core/src/series/implementations/date.rs +++ b/crates/polars-core/src/series/implementations/date.rs @@ -277,8 +277,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/datetime.rs b/crates/polars-core/src/series/implementations/datetime.rs index de22804a5ab0..f5b704a3590a 100644 --- a/crates/polars-core/src/series/implementations/datetime.rs +++ b/crates/polars-core/src/series/implementations/datetime.rs @@ -292,8 +292,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/decimal.rs b/crates/polars-core/src/series/implementations/decimal.rs index f420a76a5036..144daabf2244 100644 --- a/crates/polars-core/src/series/implementations/decimal.rs +++ b/crates/polars-core/src/series/implementations/decimal.rs @@ -322,8 +322,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } fn is_null(&self) -> BooleanChunked { diff --git a/crates/polars-core/src/series/implementations/duration.rs b/crates/polars-core/src/series/implementations/duration.rs index 6667b69fea41..2e4565015621 100644 --- a/crates/polars-core/src/series/implementations/duration.rs +++ b/crates/polars-core/src/series/implementations/duration.rs @@ -404,8 +404,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/floats.rs b/crates/polars-core/src/series/implementations/floats.rs index 3dc1458cdc17..3f6ccbe1c3f9 100644 --- a/crates/polars-core/src/series/implementations/floats.rs +++ b/crates/polars-core/src/series/implementations/floats.rs @@ -288,8 +288,8 @@ macro_rules! impl_dyn_series { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/list.rs b/crates/polars-core/src/series/implementations/list.rs index ae7c99996f36..a67dc8e8f487 100644 --- a/crates/polars-core/src/series/implementations/list.rs +++ b/crates/polars-core/src/series/implementations/list.rs @@ -147,8 +147,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/mod.rs b/crates/polars-core/src/series/implementations/mod.rs index a6afabdbe16b..d50f84c2d532 100644 --- a/crates/polars-core/src/series/implementations/mod.rs +++ b/crates/polars-core/src/series/implementations/mod.rs @@ -390,8 +390,8 @@ macro_rules! impl_dyn_series { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/null.rs b/crates/polars-core/src/series/implementations/null.rs index ddf5f746cb18..e935cb03ced8 100644 --- a/crates/polars-core/src/series/implementations/null.rs +++ b/crates/polars-core/src/series/implementations/null.rs @@ -187,8 +187,8 @@ impl SeriesTrait for NullChunked { self.length as usize } - fn has_validity(&self) -> bool { - true + fn has_nulls(&self) -> bool { + self.len() > 0 } fn rechunk(&self) -> Series { diff --git a/crates/polars-core/src/series/implementations/object.rs b/crates/polars-core/src/series/implementations/object.rs index 0784098e9460..8ba9eec2e8df 100644 --- a/crates/polars-core/src/series/implementations/object.rs +++ b/crates/polars-core/src/series/implementations/object.rs @@ -178,8 +178,8 @@ where ObjectChunked::null_count(&self.0) } - fn has_validity(&self) -> bool { - ObjectChunked::has_validity(&self.0) + fn has_nulls(&self) -> bool { + ObjectChunked::has_nulls(&self.0) } fn unique(&self) -> PolarsResult { diff --git a/crates/polars-core/src/series/implementations/string.rs b/crates/polars-core/src/series/implementations/string.rs index bffb1e27d893..07e2ec5b14ad 100644 --- a/crates/polars-core/src/series/implementations/string.rs +++ b/crates/polars-core/src/series/implementations/string.rs @@ -200,8 +200,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/implementations/struct__.rs b/crates/polars-core/src/series/implementations/struct__.rs index 32099ba0fbe2..c2c9c6f3726d 100644 --- a/crates/polars-core/src/series/implementations/struct__.rs +++ b/crates/polars-core/src/series/implementations/struct__.rs @@ -205,8 +205,8 @@ impl SeriesTrait for SeriesWrap { Ok(IdxCa::from_vec(self.name(), first)) } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } fn is_null(&self) -> BooleanChunked { diff --git a/crates/polars-core/src/series/implementations/time.rs b/crates/polars-core/src/series/implementations/time.rs index 159f07a48426..5f4df072b30c 100644 --- a/crates/polars-core/src/series/implementations/time.rs +++ b/crates/polars-core/src/series/implementations/time.rs @@ -245,8 +245,8 @@ impl SeriesTrait for SeriesWrap { self.0.null_count() } - fn has_validity(&self) -> bool { - self.0.has_validity() + fn has_nulls(&self) -> bool { + self.0.has_nulls() } #[cfg(feature = "algorithm_group_by")] diff --git a/crates/polars-core/src/series/series_trait.rs b/crates/polars-core/src/series/series_trait.rs index 0b68d451ce3c..03a9384d4f50 100644 --- a/crates/polars-core/src/series/series_trait.rs +++ b/crates/polars-core/src/series/series_trait.rs @@ -364,9 +364,8 @@ pub trait SeriesTrait: /// Count the null values. fn null_count(&self) -> usize; - /// Return if any the chunks in this `[ChunkedArray]` have a validity bitmap. - /// no bitmap means no null values. - fn has_validity(&self) -> bool; + /// Return if any the chunks in this `[ChunkedArray]` have nulls. + fn has_nulls(&self) -> bool; /// Get unique values in the Series. fn unique(&self) -> PolarsResult { diff --git a/crates/polars-ops/src/chunked_array/list/min_max.rs b/crates/polars-ops/src/chunked_array/list/min_max.rs index 7494745992e9..10f275f32183 100644 --- a/crates/polars-ops/src/chunked_array/list/min_max.rs +++ b/crates/polars-ops/src/chunked_array/list/min_max.rs @@ -1,5 +1,6 @@ use arrow::array::{Array, PrimitiveArray}; use arrow::bitmap::Bitmap; +use arrow::compute::utils::combine_validities_and; use arrow::types::NativeType; use polars_compute::min_max::MinMaxKernel; use polars_core::prelude::*; @@ -36,16 +37,9 @@ where { let values = arr.as_any().downcast_ref::>().unwrap(); let values = values.values().as_slice(); - let mut out = min_between_offsets(values, offsets); - - if let Some(validity) = validity { - if out.has_validity() { - out.apply_validity(|other_validity| validity & &other_validity) - } else { - out = out.with_validity(Some(validity.clone())); - } - } - Box::new(out) + let out = min_between_offsets(values, offsets); + let new_validity = combine_validities_and(out.validity(), validity); + out.with_validity(new_validity).to_boxed() } fn min_list_numerical(ca: &ListChunked, inner_type: &DataType) -> Series { @@ -148,7 +142,7 @@ where let mut out = max_between_offsets(values, offsets); if let Some(validity) = validity { - if out.has_validity() { + if out.null_count() > 0 { out.apply_validity(|other_validity| validity & &other_validity) } else { out = out.with_validity(Some(validity.clone())); diff --git a/crates/polars-ops/src/chunked_array/list/sum_mean.rs b/crates/polars-ops/src/chunked_array/list/sum_mean.rs index 94255979ac19..edbe584c436a 100644 --- a/crates/polars-ops/src/chunked_array/list/sum_mean.rs +++ b/crates/polars-ops/src/chunked_array/list/sum_mean.rs @@ -2,6 +2,7 @@ use std::ops::Div; use arrow::array::{Array, PrimitiveArray}; use arrow::bitmap::Bitmap; +use arrow::compute::utils::combine_validities_and; use arrow::types::NativeType; use polars_core::export::num::{NumCast, ToPrimitive}; @@ -137,15 +138,9 @@ where { let values = arr.as_any().downcast_ref::>().unwrap(); let values = values.values().as_slice(); - let mut out = mean_between_offsets::<_, S>(values, offsets); - if let Some(validity) = validity { - if out.has_validity() { - out.apply_validity(|other_validity| validity & &other_validity) - } else { - out = out.with_validity(Some(validity.clone())); - } - } - Box::new(out) + let out = mean_between_offsets::<_, S>(values, offsets); + let new_validity = combine_validities_and(out.validity(), validity); + out.with_validity(new_validity).to_boxed() } pub(super) fn mean_list_numerical(ca: &ListChunked, inner_type: &DataType) -> Series { diff --git a/crates/polars-ops/src/chunked_array/nan_propagating_aggregate.rs b/crates/polars-ops/src/chunked_array/nan_propagating_aggregate.rs index 314ca4e64ab1..fc996ceb5184 100644 --- a/crates/polars-ops/src/chunked_array/nan_propagating_aggregate.rs +++ b/crates/polars-ops/src/chunked_array/nan_propagating_aggregate.rs @@ -71,7 +71,7 @@ where } else if idx.len() == 1 { ca.get(first as usize) } else { - match (ca.has_validity(), ca.chunks().len()) { + match (ca.has_nulls(), ca.chunks().len()) { (false, 1) => take_agg_no_null_primitive_iter_unchecked( ca.downcast_iter().next().unwrap(), idx.iter().map(|i| *i as usize), @@ -140,7 +140,7 @@ where } else if idx.len() == 1 { ca.get(first as usize) } else { - match (ca.has_validity(), ca.chunks().len()) { + match (ca.has_nulls(), ca.chunks().len()) { (false, 1) => take_agg_no_null_primitive_iter_unchecked( ca.downcast_iter().next().unwrap(), idx.iter().map(|i| *i as usize), diff --git a/crates/polars-ops/src/series/ops/interpolation/interpolate.rs b/crates/polars-ops/src/series/ops/interpolation/interpolate.rs index cffbe59f5f05..11af19651fe0 100644 --- a/crates/polars-ops/src/series/ops/interpolation/interpolate.rs +++ b/crates/polars-ops/src/series/ops/interpolation/interpolate.rs @@ -48,7 +48,7 @@ where { // This implementation differs from pandas as that boundary None's are not removed. // This prevents a lot of errors due to expressions leading to different lengths. - if !chunked_arr.has_validity() || chunked_arr.null_count() == chunked_arr.len() { + if !chunked_arr.has_nulls() || chunked_arr.null_count() == chunked_arr.len() { return chunked_arr.clone(); } diff --git a/crates/polars-ops/src/series/ops/interpolation/interpolate_by.rs b/crates/polars-ops/src/series/ops/interpolation/interpolate_by.rs index 7d76f7073cd5..674cbab514e9 100644 --- a/crates/polars-ops/src/series/ops/interpolation/interpolate_by.rs +++ b/crates/polars-ops/src/series/ops/interpolation/interpolate_by.rs @@ -92,7 +92,7 @@ where { // This implementation differs from pandas as that boundary None's are not removed. // This prevents a lot of errors due to expressions leading to different lengths. - if !chunked_arr.has_validity() || chunked_arr.null_count() == chunked_arr.len() { + if !chunked_arr.has_nulls() || chunked_arr.null_count() == chunked_arr.len() { return Ok(chunked_arr.clone()); } @@ -174,7 +174,7 @@ where { // This implementation differs from pandas as that boundary None's are not removed. // This prevents a lot of errors due to expressions leading to different lengths. - if !ca.has_validity() || ca.null_count() == ca.len() { + if !ca.has_nulls() || ca.null_count() == ca.len() { return Ok(ca.clone()); } diff --git a/py-polars/polars/series/series.py b/py-polars/polars/series/series.py index bd3d6a45a43f..f298b793c9ad 100644 --- a/py-polars/polars/series/series.py +++ b/py-polars/polars/series/series.py @@ -3444,7 +3444,7 @@ def has_nulls(self) -> bool: >>> s[:2].has_nulls() False """ - return self.null_count() > 0 + return self._s.has_nulls() @deprecate_function( "Use `has_nulls` instead to check for the presence of null values.", @@ -3452,23 +3452,12 @@ def has_nulls(self) -> bool: ) def has_validity(self) -> bool: """ - Return True if the Series has a validity bitmask. + Check whether the Series contains one or more null values. .. deprecated:: 0.20.30 Use :meth:`has_nulls` instead. - - If there is no mask, it means that there are no `null` values. - - Notes - ----- - While the *absence* of a validity bitmask guarantees that a Series does not - have `null` values, the converse is not true, eg: the *presence* of a - bitmask does not mean that there are null values, as every value of the - bitmask could be `false`. - - To confirm that a column has `null` values use :meth:`has_nulls`. """ - return self._s.has_validity() + return self._s.has_nulls() def is_empty(self) -> bool: """ diff --git a/py-polars/src/map/series.rs b/py-polars/src/map/series.rs index 54f4dd3bb610..c4a4288d9d97 100644 --- a/py-polars/src/map/series.rs +++ b/py-polars/src/map/series.rs @@ -261,7 +261,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { first_value: AnyValue<'a>, ) -> PyResult { let skip = 1; - if !self.has_validity() { + if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -290,7 +290,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -329,7 +329,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -368,7 +368,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -412,7 +412,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -489,7 +489,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -554,7 +554,7 @@ where first_value: AnyValue<'a>, ) -> PyResult { let skip = 1; - if !self.has_validity() { + if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -583,7 +583,7 @@ where let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -622,7 +622,7 @@ where let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -661,7 +661,7 @@ where let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -703,7 +703,7 @@ where let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -780,7 +780,7 @@ where let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -840,7 +840,7 @@ impl<'a> ApplyLambda<'a> for StringChunked { first_value: AnyValue<'a>, ) -> PyResult { let skip = 1; - if !self.has_validity() { + if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -869,7 +869,7 @@ impl<'a> ApplyLambda<'a> for StringChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -908,7 +908,7 @@ impl<'a> ApplyLambda<'a> for StringChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -947,7 +947,7 @@ impl<'a> ApplyLambda<'a> for StringChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -988,7 +988,7 @@ impl<'a> ApplyLambda<'a> for StringChunked { let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1065,7 +1065,7 @@ impl<'a> ApplyLambda<'a> for StringChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1165,7 +1165,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { let skip = 1; // get the pypolars module let pypolars = PyModule::import_bound(py, "polars")?; - if !self.has_validity() { + if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1217,7 +1217,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { let pypolars = PyModule::import_bound(py, "polars")?; if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1277,7 +1277,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { let pypolars = PyModule::import_bound(py, "polars")?; if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1339,7 +1339,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1401,7 +1401,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1488,7 +1488,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { let pypolars = PyModule::import_bound(py, "polars")?; if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1580,7 +1580,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { let skip = 1; // get the pypolars module let pypolars = PyModule::import_bound(py, "polars")?; - if !self.has_validity() { + if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1632,7 +1632,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { let pypolars = PyModule::import_bound(py, "polars")?; if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1692,7 +1692,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { let pypolars = PyModule::import_bound(py, "polars")?; if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1754,7 +1754,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1816,7 +1816,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -1903,7 +1903,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { let pypolars = PyModule::import_bound(py, "polars")?; if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -2008,7 +2008,7 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -2047,7 +2047,7 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -2086,7 +2086,7 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -2128,7 +2128,7 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) @@ -2205,7 +2205,7 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name(), self.len())) - } else if !self.has_validity() { + } else if !self.has_nulls() { let it = self .into_no_null_iter() .skip(init_null_count + skip) diff --git a/py-polars/src/series/mod.rs b/py-polars/src/series/mod.rs index 97250530845c..fd15537dc9d5 100644 --- a/py-polars/src/series/mod.rs +++ b/py-polars/src/series/mod.rs @@ -322,8 +322,8 @@ impl PySeries { Ok(self.series.null_count()) } - fn has_validity(&self) -> bool { - self.series.has_validity() + fn has_nulls(&self) -> bool { + self.series.has_nulls() } fn equals(