Skip to content

Commit

Permalink
Update SZ3 submodule to v3.2.1
Browse files Browse the repository at this point in the history
Update SZ3 to include fix for lossless compression

Remove interp_block_size parameter
  • Loading branch information
juntyr authored and rroohhh committed Dec 9, 2024
1 parent 6935210 commit 71faf50
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 50 deletions.
53 changes: 7 additions & 46 deletions src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
#[derive(Clone, Debug, Copy)]
pub enum CompressionAlgorithm {
Interpolation {
interp_block_size: u32,
},
InterpolationLorenzo {
interp_block_size: u32,
},
Interpolation,
InterpolationLorenzo,
LorenzoRegression {
lorenzo: bool,
lorenzo_second_order: bool,
Expand All @@ -18,12 +14,8 @@ pub enum CompressionAlgorithm {
impl CompressionAlgorithm {
fn decode(config: sz3_sys::SZ3_Config) -> Self {
match config.cmprAlgo as _ {
sz3_sys::SZ3_ALGO_ALGO_INTERP => Self::Interpolation {
interp_block_size: config.interpBlockSize as _,
},
sz3_sys::SZ3_ALGO_ALGO_INTERP_LORENZO => Self::InterpolationLorenzo {
interp_block_size: config.interpBlockSize as _,
},
sz3_sys::SZ3_ALGO_ALGO_INTERP => Self::Interpolation,
sz3_sys::SZ3_ALGO_ALGO_INTERP_LORENZO => Self::InterpolationLorenzo,
sz3_sys::SZ3_ALGO_ALGO_LORENZO_REG => Self::LorenzoRegression {
lorenzo: config.lorenzo,
lorenzo_second_order: config.lorenzo2,
Expand All @@ -43,14 +35,6 @@ impl CompressionAlgorithm {
}) as _
}

fn interp_block_size(&self) -> u32 {
match self {
Self::Interpolation { interp_block_size }
| Self::InterpolationLorenzo { interp_block_size } => *interp_block_size,
_ => 32,
}
}

fn lorenzo(&self) -> bool {
match self {
Self::LorenzoRegression { lorenzo, .. } => *lorenzo,
Expand Down Expand Up @@ -97,26 +81,6 @@ impl CompressionAlgorithm {
}
}

pub fn interpolation() -> Self {
Self::Interpolation {
interp_block_size: 32,
}
}

pub fn interpolation_custom(interp_block_size: u32) -> Self {
Self::Interpolation { interp_block_size }
}

pub fn interpolation_lorenzo() -> Self {
Self::InterpolationLorenzo {
interp_block_size: 32,
}
}

pub fn interpolation_lorenzo_custom(interp_block_size: u32) -> Self {
Self::InterpolationLorenzo { interp_block_size }
}

pub fn lorenzo_regression() -> Self {
Self::LorenzoRegression {
lorenzo: true,
Expand Down Expand Up @@ -156,7 +120,7 @@ impl CompressionAlgorithm {

impl Default for CompressionAlgorithm {
fn default() -> Self {
CompressionAlgorithm::interpolation_lorenzo()
CompressionAlgorithm::InterpolationLorenzo
}
}

Expand Down Expand Up @@ -738,7 +702,6 @@ pub fn compress_with_config<V: SZ3Compressible>(
lorenzo2: config.compression_algorithm.lorenzo_second_order(),
regression: config.compression_algorithm.regression(),
regression2: config.compression_algorithm.regression_second_order(),
interpBlockSize: config.compression_algorithm.interp_block_size() as _,
pred_dim: config
.compression_algorithm
.encode_prediction_dimension(data.dims().len() as _) as _,
Expand Down Expand Up @@ -996,10 +959,8 @@ mod tests {
})
],
([
(interpolation, CompressionAlgorithm::interpolation()),
(interpolation_8, CompressionAlgorithm::interpolation_custom(8)),
(interpolation_lorenzo, CompressionAlgorithm::interpolation_lorenzo()),
(interpolation_lorenzo_8, CompressionAlgorithm::interpolation_lorenzo_custom(8)),
(interpolation, CompressionAlgorithm::Interpolation),
(interpolation_lorenzo, CompressionAlgorithm::InterpolationLorenzo),
(lorenzo_reg, CompressionAlgorithm::lorenzo_regression()),
(lorenzo_reg_all, CompressionAlgorithm::lorenzo_regression_custom(
Some(true),
Expand Down
2 changes: 1 addition & 1 deletion sz3-sys/SZ3
Submodule SZ3 updated 93 files
+164 −0 .clang-format
+19 −9 CMakeLists.txt
+15 −10 README.md
+165 −0 include/SZ3/api/impl/SZAlgoInterp.hpp
+114 −0 include/SZ3/api/impl/SZAlgoLorenzoReg.hpp
+36 −0 include/SZ3/api/impl/SZAlgoNopred.hpp
+88 −34 include/SZ3/api/impl/SZDispatcher.hpp
+18 −24 include/SZ3/api/impl/SZImpl.hpp
+110 −109 include/SZ3/api/impl/SZImplOMP.hpp
+0 −173 include/SZ3/api/impl/SZInterp.hpp
+0 −120 include/SZ3/api/impl/SZLorenzoReg.hpp
+76 −56 include/SZ3/api/sz.hpp
+27 −38 include/SZ3/compressor/Compressor.hpp
+0 −103 include/SZ3/compressor/SZGeneralCompressor.hpp
+101 −0 include/SZ3/compressor/SZGenericCompressor.hpp
+0 −482 include/SZ3/compressor/SZInterpolationCompressor.hpp
+190 −0 include/SZ3/compressor/SZIterateCompressor.hpp
+0 −78 include/SZ3/compressor/SZTruncateCompressor.hpp
+0 −519 include/SZ3/compressor/deprecated/SZBlockInterpolationCompressor.hpp
+489 −0 include/SZ3/compressor/specialized/SZBlockInterpolationCompressor.hpp
+213 −0 include/SZ3/compressor/specialized/SZExaaltCompressor.hpp
+73 −0 include/SZ3/compressor/specialized/SZTruncateCompressor.hpp
+65 −0 include/SZ3/decomposition/Decomposition.hpp
+410 −0 include/SZ3/decomposition/InterpolationDecomposition.hpp
+548 −0 include/SZ3/decomposition/LorenzoRegressionDecomposition.hpp
+52 −0 include/SZ3/decomposition/NoPredictionDecomposition.hpp
+204 −0 include/SZ3/decomposition/TimeSeriesDecomposition.hpp
+5 −7 include/SZ3/def.hpp
+617 −633 include/SZ3/encoder/ArithmeticEncoder.hpp
+29 −33 include/SZ3/encoder/BypassEncoder.hpp
+64 −69 include/SZ3/encoder/Encoder.hpp
+571 −590 include/SZ3/encoder/HuffmanEncoder.hpp
+46 −52 include/SZ3/encoder/RunlengthEncoder.hpp
+0 −72 include/SZ3/frontend/Frontend.hpp
+0 −621 include/SZ3/frontend/SZFastFrontend.hpp
+0 −150 include/SZ3/frontend/SZGeneralFrontend.hpp
+27 −28 include/SZ3/lossless/Lossless.hpp
+18 −21 include/SZ3/lossless/Lossless_bypass.hpp
+35 −44 include/SZ3/lossless/Lossless_zstd.hpp
+221 −231 include/SZ3/predictor/ComposedPredictor.hpp
+135 −143 include/SZ3/predictor/LorenzoPredictor.hpp
+129 −168 include/SZ3/predictor/MetaLorenzoPredictor.hpp
+207 −219 include/SZ3/predictor/MetaRegressionPredictor.hpp
+264 −261 include/SZ3/predictor/PolyRegressionPredictor.hpp
+105 −102 include/SZ3/predictor/Predictor.hpp
+197 −205 include/SZ3/predictor/RegressionPredictor.hpp
+4 −4 include/SZ3/preprocessor/Wavelet.hpp
+0 −204 include/SZ3/quantizer/IntegerQuantizer.hpp
+166 −0 include/SZ3/quantizer/LinearQuantizer.hpp
+51 −62 include/SZ3/quantizer/Quantizer.hpp
+236 −204 include/SZ3/utils/ByteUtil.hpp
+285 −155 include/SZ3/utils/Config.hpp
+210 −204 include/SZ3/utils/Extraction.hpp
+65 −64 include/SZ3/utils/FileUtil.hpp
+66 −66 include/SZ3/utils/Interpolators.hpp
+250 −301 include/SZ3/utils/Iterator.hpp
+368 −0 include/SZ3/utils/KmeansUtil.hpp
+51 −47 include/SZ3/utils/MemoryUtil.hpp
+88 −87 include/SZ3/utils/MetaDef.hpp
+122 −121 include/SZ3/utils/QuantOptimizatioin.hpp
+132 −134 include/SZ3/utils/Statistic.hpp
+24 −27 include/SZ3/utils/Timer.hpp
+11 −11 include/SZ3/utils/inih/INIReader.h
+3 −0 include/SZ3/utils/ska_hash/unordered_map.hpp
+21 −1 include/SZ3/version.hpp.in
+10 −12 tools/H5Z-SZ3/CMakeLists.txt
+0 −26 tools/H5Z-SZ3/README
+63 −0 tools/H5Z-SZ3/README.md
+24 −71 tools/H5Z-SZ3/include/H5Z_SZ3.hpp
+145 −1,008 tools/H5Z-SZ3/src/H5Z_SZ3.cpp
+2 −1 tools/H5Z-SZ3/test/CMakeLists.txt
+154 −204 tools/H5Z-SZ3/test/convertBinToHDF5.cpp
+248 −276 tools/H5Z-SZ3/test/dsz3FromHDF5.cpp
+19 −0 tools/H5Z-SZ3/test/h5repack.sh
+51 −213 tools/H5Z-SZ3/test/print_h5repack_args.cpp
+142 −465 tools/H5Z-SZ3/test/sz3ToHDF5.cpp
+0 −226 tools/mdz/include/ExaaltCompressor.hpp
+0 −391 tools/mdz/include/KmeansUtil.hpp
+0 −221 tools/mdz/include/TimeBasedFrontend.hpp
+231 −230 tools/mdz/include/mdz.hpp
+16 −18 tools/mdz/mdz.cpp
+3 −7 tools/mdz/mdz_smoke_test.cpp
+2 −1 tools/sz3/CMakeLists.txt
+10 −0 tools/sz3/demo/CMakeLists.txt
+113 −0 tools/sz3/demo/sz3demo_customized.cpp
+75 −0 tools/sz3/demo/sz3demo_interpolation.cpp
+84 −0 tools/sz3/demo/sz3demo_lorenzo_fast.cpp
+81 −0 tools/sz3/demo/sz3demo_lorenzo_generic.cpp
+3 −3 tools/sz3/deprecated/beta_sz_pw.cpp
+1 −1 tools/sz3/deprecated/beta_sz_truncate.cpp
+51 −63 tools/sz3/sz3.cpp
+11 −10 tools/sz3/sz3_smoke_test.cpp
+8 −8 tools/sz3c/src/sz3c.cpp
3 changes: 0 additions & 3 deletions sz3-sys/wrapper.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ struct SZ3_Config {
uint8_t lossless;
uint8_t encoder;
uint8_t interpAlgo;
int interpBlockSize;
int quantbinCnt;
int blockSize;
int stride;
Expand All @@ -43,7 +42,6 @@ struct SZ3_Config {
conf.lossless = lossless;
conf.encoder = encoder;
conf.interpAlgo = interpAlgo;
conf.interpBlockSize = interpBlockSize;
conf.quantbinCnt = quantbinCnt;
conf.blockSize = blockSize;
conf.stride = stride;
Expand All @@ -70,7 +68,6 @@ struct SZ3_Config {
lossless = conf.lossless;
encoder = conf.encoder;
interpAlgo = conf.interpAlgo;
interpBlockSize = conf.interpBlockSize;
quantbinCnt = conf.quantbinCnt;
blockSize = conf.blockSize;
stride = conf.stride;
Expand Down

0 comments on commit 71faf50

Please sign in to comment.