Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(rust): string to decimal parsing #10712

Merged
merged 6 commits into from
Aug 29, 2023
Merged

fix(rust): string to decimal parsing #10712

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d3c30ce
add check for non-numeric characters
c-peters Aug 24, 2023
51675b1
better way of checking numeric
c-peters Aug 24, 2023
11ff7c2
fix option handling
c-peters Aug 24, 2023
ba7a332
fix failing case
c-peters Aug 24, 2023
984519f
fmt
ritchie46 Aug 29, 2023
530ad83
simplify match
c-peters Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 88 additions & 59 deletions crates/polars-arrow/src/compute/decimal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use atoi::atoi;
use atoi::FromRadix10SignedChecked;

fn significant_digits(bytes: &[u8]) -> u8 {
(bytes.len() as u8) - leading_zeros(bytes)
Expand All @@ -9,12 +9,17 @@ fn leading_zeros(bytes: &[u8]) -> u8 {
}

fn split_decimal_bytes(bytes: &[u8]) -> (Option<&[u8]>, Option<&[u8]>) {
let mut a = bytes.split(|x| *x == b'.');
let mut a = bytes.splitn(2, |x| *x == b'.');
let lhs = a.next();
let rhs = a.next();
(lhs, rhs)
}

fn parse_integer_checked(bytes: &[u8]) -> Option<i128> {
let (n, len) = i128::from_radix_10_signed_checked(bytes);
n.filter(|_| len == bytes.len())
}

pub fn infer_scale(bytes: &[u8]) -> Option<u8> {
let (_lhs, rhs) = split_decimal_bytes(bytes);
rhs.map(significant_digits)
Expand All @@ -26,67 +31,64 @@ pub fn infer_scale(bytes: &[u8]) -> Option<u8> {
pub(super) fn deserialize_decimal(bytes: &[u8], precision: Option<u8>, scale: u8) -> Option<i128> {
let (lhs, rhs) = split_decimal_bytes(bytes);
let precision = precision.unwrap_or(u8::MAX);
match (lhs, rhs) {
(Some(lhs), Some(rhs)) => atoi::<i128>(lhs).and_then(|x| {
atoi::<i128>(rhs)
.map(|y| (x, lhs, y, rhs))
.and_then(|(lhs, lhs_b, rhs, rhs_b)| {
let lhs_s = significant_digits(lhs_b);
let leading_zeros_rhs = leading_zeros(rhs_b);
let rhs_s = rhs_b.len() as u8 - leading_zeros_rhs;

// parameters don't match bytes
if lhs_s + rhs_s > precision || rhs_s > scale {
None
}
// significant digits don't fit scale
else if rhs_s < scale {
// scale: 2
// number: x.09
// significant digits: 1
// leading_zeros: 1
// parsed: 9
// so this is correct
if leading_zeros_rhs + rhs_s == scale {
Some((lhs, rhs))

let lhs_b = lhs?;
parse_integer_checked(lhs_b).and_then(|x| {
match rhs {
Some(rhs) => {
parse_integer_checked(rhs)
.map(|y| (x, lhs_b, y, rhs))
.and_then(|(lhs, lhs_b, rhs, rhs_b)| {
let lhs_s = significant_digits(lhs_b);
let leading_zeros_rhs = leading_zeros(rhs_b);
let rhs_s = rhs_b.len() as u8 - leading_zeros_rhs;

// parameters don't match bytes
if lhs_s + rhs_s > precision || rhs_s > scale {
None
}
// significant digits don't fit scale
else if rhs_s < scale {
// scale: 2
// number: x.09
// significant digits: 1
// leading_zeros: 1
// parsed: 9
// so this is correct
if leading_zeros_rhs + rhs_s == scale {
Some((lhs, rhs))
}
// scale: 2
// number: x.9
// significant digits: 1
// parsed: 9
// so we must multiply by 10 to get 90
else {
let diff = scale as u32 - (rhs_s + leading_zeros_rhs) as u32;
Some((lhs, rhs * 10i128.pow(diff)))
}
}
// scale: 2
// number: x.9
// significant digits: 1
// parsed: 9
// so we must multiply by 10 to get 90
// number: x.90
// significant digits: 2
// parsed: 90
// so this is correct
else {
let diff = scale as u32 - (rhs_s + leading_zeros_rhs) as u32;
Some((lhs, rhs * 10i128.pow(diff)))
Some((lhs, rhs))
}
}
// scale: 2
// number: x.90
// significant digits: 2
// parsed: 90
// so this is correct
else {
Some((lhs, rhs))
}
})
.map(|(lhs, rhs)| {
lhs * 10i128.pow(scale as u32) + (if lhs < 0 { -rhs } else { rhs })
})
}),
(None, Some(rhs)) => {
if rhs.len() > precision as usize || rhs.len() != scale as usize {
return None;
}
atoi::<i128>(rhs)
},
(Some(lhs), None) => {
if lhs.len() > precision as usize || scale != 0 {
return None;
}
atoi::<i128>(lhs)
},
(None, None) => None,
}
})
.map(|(lhs, rhs)| {
lhs * 10i128.pow(scale as u32) + (if lhs < 0 { -rhs } else { rhs })
})
},
None => {
if lhs_b.len() > precision as usize || scale != 0 {
return None;
}
parse_integer_checked(lhs_b)
},
}
})
}

#[cfg(test)]
Expand Down Expand Up @@ -127,5 +129,32 @@ mod test {
deserialize_decimal(val.as_bytes(), precision, scale),
Some(1000000000000000000)
);
let scale = 5;
let val = "12ABC.34";
assert_eq!(deserialize_decimal(val.as_bytes(), precision, scale), None);

let val = "1ABC2.34";
assert_eq!(deserialize_decimal(val.as_bytes(), precision, scale), None);

let val = "12.3ABC4";
assert_eq!(deserialize_decimal(val.as_bytes(), precision, scale), None);

let val = "12.3.ABC4";
assert_eq!(deserialize_decimal(val.as_bytes(), precision, scale), None);

let val = "";
assert_eq!(deserialize_decimal(val.as_bytes(), precision, scale), None);

let val = "5.";
assert_eq!(
deserialize_decimal(val.as_bytes(), precision, scale),
Some(500000i128)
);

let val = ".5";
assert_eq!(
deserialize_decimal(val.as_bytes(), precision, scale),
Some(50000i128)
);
}
}