diff --git a/Cargo.toml b/Cargo.toml index f46608fa..46151bef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brotli" -version = "3.0.3" +version = "3.1.0" authors = ["Daniel Reiter Horn ", "The Brotli Authors"] description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe." license = "BSD-3-Clause/MIT" diff --git a/c/Cargo.toml b/c/Cargo.toml index a650b059..3c6ec764 100644 --- a/c/Cargo.toml +++ b/c/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brotli-ffi" -version = "1.0.1" +version = "1.1.0" authors = ["Daniel Reiter Horn ", "The Brotli Authors"] description = "A brotli compressor and decompressor that with an interface exactly matching https://github.com/google/brotli. All included code is safe except the thin exported C-compatible functions." license = "BSD-3-Clause/MIT" @@ -19,7 +19,7 @@ crate-type=["cdylib", "staticlib", "rlib"] lto=true [dependencies] -"brotli" = {version="~3.0", default-features=false} +"brotli" = {version="~3.1", default-features=false} [features] validation=["brotli/validation"] diff --git a/c/brotli/broccoli.h b/c/brotli/broccoli.h index 9560afc0..cee94a8b 100644 --- a/c/brotli/broccoli.h +++ b/c/brotli/broccoli.h @@ -32,7 +32,17 @@ BroccoliResult BroccoliConcatStream( size_t *available_out, uint8_t **output_buf_ptr); +BroccoliResult BroccoliConcatStreaming( + BroccoliState *state, + size_t *available_in, + const uint8_t *input_buf_ptr, + size_t *available_out, + uint8_t *output_buf_ptr); + BroccoliResult BroccoliConcatFinish(BroccoliState * state, size_t *available_out, uint8_t**output_buf); +BroccoliResult BroccoliConcatFinished(BroccoliState * state, + size_t *available_out, + uint8_t*output_buf); #endif diff --git a/c/brotli/decode.h b/c/brotli/decode.h index 9305276c..47edf838 100644 --- a/c/brotli/decode.h +++ b/c/brotli/decode.h @@ -118,6 +118,21 @@ typedef enum { #undef BROTLI_ERROR_CODE_ENUM_ITEM_ #undef BROTLI_COMMA_ +typedef struct HuffmanCodeStruct { + uint16_t value; + uint8_t bits; +} HuffmanCode; + + + +typedef struct BrotliDecoderReturnInfoStruct { + size_t decoded_size; + char error[256]; + BrotliDecoderResult result; + BrotliDecoderErrorCode code; +} BrotliDecoderReturnInfo; + + /** * The value of the last error code, negative integer. * @@ -205,6 +220,25 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress( size_t* decoded_size, uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]); +BROTLI_DEC_API BrotliDecoderReturnInfo BrotliDecoderDecompressWithReturnInfo( + size_t encoded_size, + const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)], + size_t decoded_size, + uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(decoded_size)]); + +BROTLI_DEC_API BrotliDecoderReturnInfo BrotliDecoderDecompressPrealloc( + size_t encoded_size, + const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)], + size_t decoded_size, + uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(decoded_size)], + size_t scratch_u8_size, + uint8_t scratch_u8_buffer[BROTLI_ARRAY_PARAM(scratch_u8_size)], + size_t scratch_u32_size, + uint32_t scratch_u32_buffer[BROTLI_ARRAY_PARAM(scratch_u32_size)], + size_t scratch_hc_size, + HuffmanCode scratch_hc_buffer[BROTLI_ARRAY_PARAM(scratch_hc_size)] + ); + /** * Decompresses the input stream to the output stream. * @@ -247,6 +281,10 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream( BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in, size_t* available_out, uint8_t** next_out, size_t* total_out); +BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStreaming( + BrotliDecoderState* state, size_t* available_in, const uint8_t* next_in, + size_t* available_out, uint8_t* next_out); + /** * Checks if decoder has more output. * diff --git a/c/brotli/encode.h b/c/brotli/encode.h index d3e22c91..8c2a0b6f 100644 --- a/c/brotli/encode.h +++ b/c/brotli/encode.h @@ -384,6 +384,10 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStream( const uint8_t** next_in, size_t* available_out, uint8_t** next_out, size_t* total_out); +BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStreaming( + BrotliEncoderState* state, BrotliEncoderOperation op, size_t* available_in, + const uint8_t* next_in, size_t* available_out, uint8_t* next_out); + /** * Checks if encoder instance reached the final state. * diff --git a/c/go/interface_test.go b/c/go/interface_test.go index 0755ab1f..c58ea011 100644 --- a/c/go/interface_test.go +++ b/c/go/interface_test.go @@ -80,6 +80,39 @@ func TestCompressRoundtrip(*testing.T) { } } +func TestCompressRoundtripMulti(*testing.T) { + tmp := testData() + data := tmp[:len(tmp)-17] + outBuffer := bytes.NewBuffer(nil) + var options = brotli.CompressionOptions{ + NumThreads: 16, + Quality: 9, + Catable: true, + Appendable: true, + Magic: true, + } + writer := brotli.NewMultiCompressionWriter( + brotli.NewDecompressionWriter( + outBuffer, + ), + options, + ) + _, err := writer.Write(data[:]) + if err != nil { + panic(err) + } + err = writer.Close() + if err != nil { + panic(err) + } + if len(outBuffer.Bytes()) == 0 { + panic("Zero output buffer") + } + if !bytes.Equal(outBuffer.Bytes(), data[:]) { + panic(fmt.Sprintf("Bytes not equal %d, %d", len(outBuffer.Bytes()), len(data))) + } +} + func TestRejectCorruptBuffers(*testing.T) { tmp := testData() data := tmp[:len(tmp)-17] diff --git a/c/py/brotli.py b/c/py/brotli.py index 348f57a5..94756842 100644 --- a/c/py/brotli.py +++ b/c/py/brotli.py @@ -1,7 +1,7 @@ import ctypes import sys -from ctypes import c_uint, pointer, POINTER, c_size_t, c_void_p, c_uint32, c_ubyte, c_char_p, byref +from ctypes import c_uint, c_int, pointer, POINTER, c_size_t, c_void_p, c_uint32, c_ubyte, c_char_p, byref class BrotliEncoderWorkPool(ctypes.Structure): pass BrotliEncoderWorkPool= ctypes.POINTER(BrotliEncoderWorkPool) @@ -15,6 +15,7 @@ class BrotliEncoderWorkPool(ctypes.Structure): brotli_library=ctypes.CDLL("target/release/libbrotli_ffi.dylib") except OSError: brotli_library=ctypes.CDLL("target/release/libbrotli_ffi.so") + _BrotliEncoderCreateWorkPool = brotli_library.BrotliEncoderCreateWorkPool _BrotliEncoderCreateWorkPool.restype = POINTER(BrotliEncoderWorkPool) _BrotliEncoderCompressWorkPool = brotli_library.BrotliEncoderCompressWorkPool @@ -23,6 +24,14 @@ class BrotliDecoderState(ctypes.Structure): pass class BrotliDecompressorException(Exception): pass +class BrotliDecoderReturnInfo(ctypes.Structure): + _fields_ = [('decoded_size', c_size_t), + ('error_string', c_ubyte * 256), + ('result', c_int), + ('error_code', c_int), + ] +BrotliDecoderDecompressWithReturnInfo = brotli_library.BrotliDecoderDecompressWithReturnInfo +BrotliDecoderDecompressWithReturnInfo.restype = BrotliDecoderReturnInfo _BrotliDecoderCreateInstance = brotli_library.BrotliDecoderCreateInstance _BrotliDecoderCreateInstance.restype = POINTER(BrotliDecoderState) _BrotliDecoderDestroyInstance = brotli_library.BrotliDecoderDestroyInstance @@ -127,20 +136,20 @@ def BrotliDecode(any_input, expected_size=4096 * 1024, max_expected_size = 256 * decoded_size = c_size_t(expected_size) decoded = (c_ubyte * decoded_size.value)() - res = brotli_library.BrotliDecoderDecompress(len(input), - input, - byref(decoded_size), - byref(decoded)) - if res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + res = BrotliDecoderDecompressWithReturnInfo(len(input), + input, + decoded_size, + byref(decoded)) + if res.result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: raise BrotliDecompressorException("EarlyEOF") - elif res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: - expected_size *= 2 - elif res == BROTLI_DECODER_RESULT_SUCCESS: - return bytearray(decoded[:decoded_size.value]) - else: + elif res.result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: expected_size *= 2 if expected_size > max_expected_size: raise BrotliDecompressorException("Brotli file > " + str(max_expected_size) + " or corrupt brotli file") + elif res.result == BROTLI_DECODER_RESULT_SUCCESS: + return bytearray(decoded[:res.decoded_size]) + else: + raise BrotliDecompressorException(''.join(chr(x) for x in res.error_string)) def BrotliCompress( diff --git a/c/py/brotli_test.py b/c/py/brotli_test.py index a72a1a20..54efe76a 100644 --- a/c/py/brotli_test.py +++ b/c/py/brotli_test.py @@ -45,6 +45,17 @@ def test_tiny_alloc(self): rt = BrotliDecode(output, 2) assert rt == self.test_data assert len(output) < 1024 * 1024 + def test_single_thread(self): + output = BrotliCompress(self.test_data, + { + BROTLI_PARAM_QUALITY:5, + BROTLI_PARAM_CATABLE:1, + BROTLI_PARAM_MAGIC_NUMBER:1, + }, + 1) + rt = BrotliDecode(output, 2) + assert rt == self.test_data + assert len(output) < 1024 * 1024 def test_memory_view(self): output = BrotliCompress(memoryview(self.test_data), { @@ -82,7 +93,7 @@ def test_rnd(self): rt = BrotliDecode(output) assert rt == random_data assert len(output) > 130000 - def test_1(self): + def test_truncation(self): output = BrotliCompress(self.test_data[:65536], { BROTLI_PARAM_QUALITY:6, @@ -100,6 +111,24 @@ def test_1(self): pass else: assert False, "Should have errored" + def test_corruption(self): + output = BrotliCompress(self.test_data[:65536], + { + BROTLI_PARAM_QUALITY:6, + BROTLI_PARAM_CATABLE:1, + BROTLI_PARAM_MAGIC_NUMBER:1, + }, + 8) + corrupt = output[:len(output)/2] + output[len(output)/2 + 1:] + rt = BrotliDecode(output) + assert rt == self.test_data[:65536] + assert len(output) < 1024 * 1024 + try: + BrotliDecode(corrupt) + except BrotliDecompressorException: + pass + else: + assert False, "Should have errored" if __name__ == '__main__': unittest.main() diff --git a/src/enc/encode.rs b/src/enc/encode.rs index 1ddd6728..0e777e16 100755 --- a/src/enc/encode.rs +++ b/src/enc/encode.rs @@ -3530,7 +3530,7 @@ pub fn BrotliEncoderCompressStream(s: &mut BrotliEncoderStateStruct) -> i32 { +pub fn BrotliEncoderIsFinished(s: &BrotliEncoderStateStruct) -> i32 { if !!((*s).stream_state_ as (i32) == BrotliEncoderStreamState::BROTLI_STREAM_FINISHED as (i32) && (BrotliEncoderHasMoreOutput(s) == 0)) { 1i32 diff --git a/src/ffi/broccoli.rs b/src/ffi/broccoli.rs index 8a616b01..a01f0c4c 100644 --- a/src/ffi/broccoli.rs +++ b/src/ffi/broccoli.rs @@ -92,6 +92,22 @@ pub unsafe extern fn BroccoliConcatStream( ret } +#[no_mangle] +pub unsafe extern fn BroccoliConcatStreaming( + state: *mut BroccoliState, + available_in: *mut usize, + mut input_buf: *const u8, + available_out: *mut usize, + mut output_buf: *mut u8) -> BroccoliResult { + BroccoliConcatStream( + state, + available_in, + &mut input_buf, + available_out, + &mut output_buf) + +} + #[no_mangle] pub unsafe extern fn BroccoliConcatFinish( state: *mut BroccoliState, @@ -106,3 +122,15 @@ pub unsafe extern fn BroccoliConcatFinish( *state = BroccoliState::from(bro_catli); ret } + +// exactly the same as BrotliConcatFinish but without the indirect +#[no_mangle] +pub unsafe extern fn BroccoliConcatFinished( + state: *mut BroccoliState, + available_out: *mut usize, + mut output_buf: *mut u8) -> BroCatliResult { + BroccoliConcatFinish( + state, + available_out, + &mut output_buf) +} diff --git a/src/ffi/compressor.rs b/src/ffi/compressor.rs index d7d0880f..3dc88d36 100755 --- a/src/ffi/compressor.rs +++ b/src/ffi/compressor.rs @@ -237,6 +237,26 @@ pub unsafe extern fn BrotliEncoderCompress( }, } } + +#[no_mangle] +pub unsafe extern fn BrotliEncoderCompressStreaming( + state_ptr: *mut BrotliEncoderState, + op: BrotliEncoderOperation, + available_in: *mut usize, + mut input_buf: *const u8, + available_out: *mut usize, + mut output_buf: *mut u8, +) -> i32 { + BrotliEncoderCompressStream(state_ptr, + op, + available_in, + &mut input_buf, + available_out, + &mut output_buf, + core::ptr::null_mut()) + +} + #[no_mangle] pub unsafe extern fn BrotliEncoderCompressStream( state_ptr: *mut BrotliEncoderState, diff --git a/src/ffi/decompressor.rs b/src/ffi/decompressor.rs index dad0b9e1..4d435f25 100644 --- a/src/ffi/decompressor.rs +++ b/src/ffi/decompressor.rs @@ -1,4 +1,5 @@ pub use brotli_decompressor::ffi; +pub use brotli_decompressor::{BrotliDecoderReturnInfo, HuffmanCode}; pub use brotli_decompressor::ffi::interface::{ brotli_alloc_func, brotli_free_func, @@ -45,6 +46,59 @@ ffi::BrotliDecoderDecompressStream( total_out) } +pub unsafe extern fn CBrotliDecoderDecompressStreaming( + state_ptr: *mut ffi::BrotliDecoderState, + available_in: *mut usize, + input_buf_ptr: *const u8, + available_out: *mut usize, + output_buf_ptr: *mut u8, +) -> ffi::interface::BrotliDecoderResult { +ffi::BrotliDecoderDecompressStreaming( + state_ptr, + available_in, + input_buf_ptr, + available_out, + output_buf_ptr) +} + +pub unsafe extern fn CBrotliDecoderDecompressWithReturnInfo( + available_in: usize, + input_buf_ptr: *const u8, + available_out_and_scratch: usize, + output_buf_and_scratch: *mut u8, +) -> BrotliDecoderReturnInfo { +ffi::BrotliDecoderDecompressWithReturnInfo( + available_in, + input_buf_ptr, + available_out_and_scratch, + output_buf_and_scratch) +} + +pub unsafe extern fn CBrotliDecoderDecompressPrealloc( + available_in: usize, + input_buf_ptr: *const u8, + available_out: usize, + output_buf_ptr: *mut u8, + available_u8: usize, + u8_ptr: *mut u8, + available_u32: usize, + u32_ptr: *mut u32, + available_hc: usize, + hc_ptr: *mut HuffmanCode, +) -> BrotliDecoderReturnInfo { +ffi::BrotliDecoderDecompressPrealloc( + available_in, + input_buf_ptr, + available_out, + output_buf_ptr, + available_u8, + u8_ptr, + available_u32, + u32_ptr, + available_hc, + hc_ptr) +} + pub unsafe extern fn CBrotliDecoderMallocU8(state_ptr: *mut ffi::BrotliDecoderState, size: usize) -> *mut u8 { ffi::BrotliDecoderMallocU8(state_ptr, size) } diff --git a/src/ffi/multicompress/mod.rs b/src/ffi/multicompress/mod.rs index ba7055ba..3236c961 100755 --- a/src/ffi/multicompress/mod.rs +++ b/src/ffi/multicompress/mod.rs @@ -9,6 +9,13 @@ use core; #[allow(unused_imports)] use brotli_decompressor; use super::compressor; +use ::enc::encode::{BrotliEncoderCreateInstance, + BrotliEncoderOperation, + BrotliEncoderSetParameter, + BrotliEncoderCompressStream, + BrotliEncoderIsFinished, + BrotliEncoderDestroyInstance, +}; use brotli_decompressor::ffi::alloc_util::SubclassableAllocator; use brotli_decompressor::ffi::interface::{ brotli_alloc_func, @@ -50,6 +57,42 @@ pub extern fn BrotliEncoderMaxCompressedSizeMulti(input_size: usize, num_threads ::enc::encode::BrotliEncoderMaxCompressedSizeMulti(input_size, num_threads) } +fn help_brotli_encoder_compress_single( + param_keys: &[BrotliEncoderParameter], + param_values: &[u32], + input: &[u8], + output: &mut[u8], + encoded_size: &mut usize, + m8: BrotliSubclassableAllocator, +) -> i32 { + let mut encoder = BrotliEncoderCreateInstance(m8); + for (p, v) in param_keys.iter().zip(param_values.iter()) { + BrotliEncoderSetParameter(&mut encoder, *p, *v); + } + let mut result; + let mut available_in = input.len(); + let mut next_in_offset = 0usize; + let mut available_out = output.len(); + let mut next_out_offset = 0usize; + let mut total_out = Some(0usize); + result = BrotliEncoderCompressStream(&mut encoder, + BrotliEncoderOperation::BROTLI_OPERATION_FINISH, + &mut available_in, + input, + &mut next_in_offset, + &mut available_out, + output, + &mut next_out_offset, + &mut total_out, + &mut |_a,_b,_c,_d|()); + if BrotliEncoderIsFinished(&encoder) == 0 { + result = 0i32; + } + *encoded_size = total_out.unwrap(); + BrotliEncoderDestroyInstance(&mut encoder); + result +} + #[no_mangle] pub unsafe extern fn BrotliEncoderCompressMulti( num_params: usize, @@ -67,22 +110,42 @@ pub unsafe extern fn BrotliEncoderCompressMulti( if desired_num_threads == 0 { return 0; } - let null_opaques = [core::ptr::null_mut::();MAX_THREADS]; - let alloc_opaque = if alloc_opaque_per_thread.is_null() { - &null_opaques[..] - } else { - slice_from_raw_parts_or_nil(alloc_opaque_per_thread, desired_num_threads) - }; - let param_keys_slice = slice_from_raw_parts_or_nil(param_keys, num_params); - let param_values_slice = slice_from_raw_parts_or_nil(param_values, num_params); - let mut params = BrotliEncoderParams::default(); - for (k,v) in param_keys_slice.iter().zip(param_values_slice.iter()) { - if set_parameter(&mut params, *k, *v) == 0 { - return 0; - } - } let num_threads = core::cmp::min(desired_num_threads, MAX_THREADS); - let mut alloc_array:[_;MAX_THREADS] = [ + match compressor::catch_panic(|| { + let param_keys_slice = slice_from_raw_parts_or_nil(param_keys, num_params); + let param_values_slice = slice_from_raw_parts_or_nil(param_values, num_params); + let input_slice = slice_from_raw_parts_or_nil(input, input_size); + let output_slice = slice_from_raw_parts_or_nil_mut(encoded, *encoded_size); + if num_threads == 1 { + let allocators = CAllocator { + alloc_func:alloc_func, + free_func:free_func, + opaque:if alloc_opaque_per_thread.is_null() {core::ptr::null_mut()} else {*alloc_opaque_per_thread}, + }; + let m8 = BrotliSubclassableAllocator::new( + SubclassableAllocator::new(allocators.clone())); + return help_brotli_encoder_compress_single( + param_keys_slice, + param_values_slice, + input_slice, + output_slice, + &mut *encoded_size, + m8, + ) + } + let null_opaques = [core::ptr::null_mut::();MAX_THREADS]; + let alloc_opaque = if alloc_opaque_per_thread.is_null() { + &null_opaques[..] + } else { + slice_from_raw_parts_or_nil(alloc_opaque_per_thread, desired_num_threads) + }; + let mut params = BrotliEncoderParams::default(); + for (k,v) in param_keys_slice.iter().zip(param_values_slice.iter()) { + if set_parameter(&mut params, *k, *v) == 0 { + return 0; + } + } + let mut alloc_array:[_;MAX_THREADS] = [ make_send_alloc!(alloc_func, free_func, alloc_opaque[0]), make_send_alloc!(alloc_func, free_func, alloc_opaque[1%desired_num_threads]), make_send_alloc!(alloc_func, free_func, alloc_opaque[2%desired_num_threads]), @@ -100,11 +163,13 @@ pub unsafe extern fn BrotliEncoderCompressMulti( make_send_alloc!(alloc_func, free_func, alloc_opaque[14%desired_num_threads]), make_send_alloc!(alloc_func, free_func, alloc_opaque[15%desired_num_threads]), ]; + + let owned_input = &mut Owned::new(SliceRef(input_slice)); let res = enc::compress_multi_no_threadpool( - ¶ms, - &mut Owned::new(SliceRef(slice_from_raw_parts_or_nil(input, input_size))), - slice_from_raw_parts_or_nil_mut(encoded, *encoded_size), - &mut alloc_array[..num_threads], + ¶ms, + owned_input, + output_slice, + &mut alloc_array[..num_threads], ); match res { Ok(size) => { @@ -115,6 +180,13 @@ pub unsafe extern fn BrotliEncoderCompressMulti( return 0; } } + }) { + Ok(ret) => return ret, + Err(panic_err) => { + error_print(panic_err); + return 0; + }, + } } #[no_mangle]