try to plumb ffi apis

dropbox · Nov 9, 2021 · df6a88e · df6a88e
1 parent 0f746e5
commit df6a88e
Show file tree

Hide file tree

Showing 8 changed files with 103 additions and 24 deletions.
diff --git a/c/arg.h b/c/arg.h
@@ -70,6 +70,21 @@ size_t set_options(BrotliEncoderParameter *out_encoder_param_keys,
             out_encoder_param_values[ret] = 1;
             ret += 1;
         }
+        if (strstr(argv[i], "-appendable") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_APPENDABLE;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
+        if (strstr(argv[i], "-bytealign") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_BYTE_ALIGN;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
+        if (strstr(argv[i], "-bare") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_BARE_STREAM;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
     }
     return ret;
 }
diff --git a/c/brotli/encode.h b/c/brotli/encode.h
@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
   BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
   BROTLI_PARAM_CATABLE = 167,
   BROTLI_PARAM_APPENDABLE = 168,
-  BROTLI_PARAM_MAGIC_NUMBER = 169
+  BROTLI_PARAM_MAGIC_NUMBER = 169,
+  BROTLI_PARAM_NO_DICTIONARY = 170,
+  BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
+  BROTLI_PARAM_BYTE_ALIGN = 172,
+  BROTLI_PARAM_BARE_STREAM = 173
 } BrotliEncoderParameter;
 
 /**

diff --git a/c/go/brotli/brotli.go b/c/go/brotli/brotli.go
@@ -43,6 +43,8 @@ type CompressionOptions struct {
 	Catable                       bool
 	Appendable                    bool
 	Magic                         bool
+	ByteAlign                     bool
+	BareStream                    bool
 	Mode                          int
 	LgWin                         byte
 	LgBlock                       byte
@@ -157,6 +159,14 @@ func makeCompressionOptionsStreams(options CompressionOptions,
 		qualityParams = append(qualityParams, C.BROTLI_PARAM_MAGIC_NUMBER)
 		values = append(values, 1)
 	}
+	if options.ByteAlign {
+		qualityParams = append(qualityParams, C.BROTLI_PARAM_BYTE_ALIGN)
+		values = append(values, 1)
+	}
+	if options.BareStream {
+		qualityParams = append(qualityParams, C.BROTLI_PARAM_BARE_STREAM)
+		values = append(values, 1)
+	}
 	if options.Mode != 0 {
 		qualityParams = append(qualityParams, C.BROTLI_PARAM_MODE)
 		values = append(values, C.uint32_t(options.Mode))

diff --git a/c/go/brotli/brotli/encode.h b/c/go/brotli/brotli/encode.h
@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
   BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
   BROTLI_PARAM_CATABLE = 167,
   BROTLI_PARAM_APPENDABLE = 168,
-  BROTLI_PARAM_MAGIC_NUMBER = 169
+  BROTLI_PARAM_MAGIC_NUMBER = 169,
+  BROTLI_PARAM_NO_DICTIONARY = 170,
+  BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
+  BROTLI_PARAM_BYTE_ALIGN = 172,
+  BROTLI_PARAM_BARE_STREAM = 173,
 } BrotliEncoderParameter;
 
 /**

diff --git a/c/py/brotli.py b/c/py/brotli.py
@@ -313,6 +313,10 @@ def BrotliParseHeader(raw_data):
 BROTLI_PARAM_CATABLE = 167
 BROTLI_PARAM_APPENDABLE = 168
 BROTLI_PARAM_MAGIC_NUMBER = 169
+BROTLI_PARAM_NO_DICTIONARY = 170
+BROTLI_PARAM_FAVOR_EFFICIENCY = 171
+BROTLI_PARAM_BYTE_ALIGN = 172
+BROTLI_PARAM_BARE_STREAM = 173
 
 #simple test binary
 def main(args):

diff --git a/src/bin/brotli.rs b/src/bin/brotli.rs
@@ -530,6 +530,7 @@ fn main() {
       }
       if (argument == "-bare" || argument == "--bare") && !double_dash {
           params.bare_stream = true;
+          params.byte_align = true;
           continue;
       }
       if (argument == "-appendable" || argument == "--appendable") && !double_dash {
@@ -753,6 +754,14 @@ fn main() {
       }
       panic!("Unknown Argument {:}", argument);
    }
+   if params.bare_stream && !params.appendable {
+      println_stderr!("bare streams only supported when catable or appendable!");
+      return;
+   }
+   if params.byte_align && !params.appendable {
+      println_stderr!("byte aligned streams only supported when catable or appendable!");
+      return;
+   }
    if filenames[0] != "" {
       let mut input = match File::open(&Path::new(&filenames[0])) {
         Err(why) => panic!("couldn't open {:}\n{:}", filenames[0], why),

diff --git a/src/enc/brotli_bit_stream.rs b/src/enc/brotli_bit_stream.rs
@@ -2765,18 +2765,51 @@ pub fn BrotliStoreUncompressedMetaBlock<Cb, Alloc: BrotliAlloc>
   }
 }
 
-
 pub fn BrotliStoreSyncMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
   BrotliWriteBits(6u8, 6u64, storage_ix, storage);
   JumpToByteBoundary(storage_ix, storage);
 }
 
+pub fn BrotliWritePaddingMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
+  if *storage_ix & 7 != 0 {
+    BrotliWriteBits(6u8, 6u64, storage_ix, storage);
+    JumpToByteBoundary(storage_ix, storage);
+  }
+}
+
 pub fn BrotliWriteEmptyLastMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
     BrotliWriteBits(1u8, 1u64, storage_ix, storage);
     BrotliWriteBits(1u8, 1u64, storage_ix, storage);
     JumpToByteBoundary(storage_ix, storage);
 }
 
+pub fn BrotliWriteRawMetadataMetaBlock(storage_ix: &mut usize, storage: &mut [u8], len: usize, data: &mut [u8]) {
+    BrotliWriteBits(1u8, 0u64, storage_ix, storage); // not last
+    BrotliWriteBits(2u8, 3u64, storage_ix, storage); // MNIBBLES = 0 (pattern 1,1)
+    BrotliWriteBits(1u8, 0u64, storage_ix, storage); // reserved
+    if len > 16777215 {
+        panic!("metadata too large");
+    } else if len > 65535 {
+        BrotliWriteBits(2u8, 3u64, storage_ix, storage);
+        BrotliWriteBits(8u8, ((len >> 16) & 255) as u64, storage_ix, storage);
+        BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage);
+        BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
+    } else if len > 255 {
+        BrotliWriteBits(2u8, 2u64, storage_ix, storage);
+        BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage);
+        BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
+    } else if len == 0 {
+        BrotliWriteBits(2u8, 0u64, storage_ix, storage);
+    } else {
+        BrotliWriteBits(2u8, 1u64, storage_ix, storage);
+        BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
+    }
+    JumpToByteBoundary(storage_ix, storage);
+    for index in 0..len {
+        BrotliWriteBits(8u8, data[index] as u64, storage_ix, storage);
+    }
+}
+
 const MAX_SIZE_ENCODING:usize = 10;
 
 fn encode_base_128(mut value: u64)-> (usize, [u8;MAX_SIZE_ENCODING]) {

diff --git a/src/enc/encode.rs b/src/enc/encode.rs
@@ -16,10 +16,10 @@ use super::bit_cost::{BitsEntropy, ShannonEntropy};
 use super::block_split::BlockSplit;
 #[allow(unused_imports)]
 use super::brotli_bit_stream::{BrotliBuildAndStoreHuffmanTreeFast, BrotliStoreHuffmanTree,
-                               BrotliStoreMetaBlock, BrotliStoreSyncMetaBlock,
-                               BrotliStoreMetaBlockFast, BrotliStoreMetaBlockTrivial,
-                               BrotliStoreUncompressedMetaBlock,
-                               BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock,
+                               BrotliStoreMetaBlock, BrotliStoreMetaBlockFast,
+                               BrotliStoreMetaBlockTrivial, BrotliStoreUncompressedMetaBlock,
+                               BrotliWriteEmptyLastMetaBlock,
+                               BrotliWritePaddingMetaBlock, BrotliWriteMetadataMetaBlock,
                                MetaBlockSplit, RecoderState, JumpToByteBoundary};
 
 use enc::input_pair::InputReferenceMut;
@@ -740,7 +740,7 @@ fn EnsureInitialized<Alloc: BrotliAlloc>
     if (*s).params.quality == 0i32 || (*s).params.quality == 1i32 {
       lgwin = brotli_max_int(lgwin, 18i32);
     }
-    if !(*s).params.bare_stream {
+    if !((*s).params.catable && (*s).params.bare_stream) {
       EncodeWindowBits(lgwin, s.params.large_window, &mut (*s).last_bytes_, &mut (*s).last_bytes_bits_);
     }
   }
@@ -1969,6 +1969,15 @@ fn DecideOverLiteralContextModeling(input: &[u8],
                      literal_context_map);
   }
 }
+fn WriteEmptyLastBlocksInternal(params: &BrotliEncoderParams, storage_ix: &mut usize, storage: &mut [u8]) {
+  // insert empty block for byte alignment if required
+  if params.byte_align {
+    BrotliWritePaddingMetaBlock(storage_ix, storage);
+  }
+  if !params.bare_stream {
+    BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
+  }
+}
 fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                           Cb>
             (alloc: &mut Alloc,
@@ -2007,8 +2016,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
   let literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
   let mut block_params = params.clone();
   if bytes == 0usize {
-    BrotliWriteBits(2usize, 3, storage_ix, storage);
-    *storage_ix = (*storage_ix).wrapping_add(7u32 as (usize)) & !7u32 as (usize);
+    WriteEmptyLastBlocksInternal(params, storage_ix, storage);
     return;
   }
   if ShouldCompress(data,
@@ -2031,13 +2039,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                                      false,
                                      cb);
     if actual_is_last != is_last {
-      // insert empty block for byte alignment if required
-      if params.byte_align && ((*storage_ix & 7u32 as (usize)) != 0) {
-        BrotliStoreSyncMetaBlock(storage_ix, storage);
-      }
-      if !params.bare_stream {
-        BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
-      }
+      WriteEmptyLastBlocksInternal(params, storage_ix, storage);
     }
     return;
   }
@@ -2177,13 +2179,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                                        cb);
   }
   if actual_is_last != is_last {
-    // insert empty block for byte alignment if required
-    if params.byte_align && ((*storage_ix & 7u32 as (usize)) != 0) {
-      BrotliStoreSyncMetaBlock(storage_ix, storage);
-    }
-    if !params.bare_stream {
-      BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
-    }
+    WriteEmptyLastBlocksInternal(params, storage_ix, storage);
   }
 }
 
@@ -2280,6 +2276,10 @@ fn EncodeData<Alloc: BrotliAlloc,
       *out_size = catable_header_size;
       s.is_first_mb = IsFirst::HeaderWritten;
     }
+    // fixup for empty stream - note: catable is always appendable
+    if bytes == 0 && s.params.byte_align && s.params.appendable && !s.params.catable {
+      BrotliWritePaddingMetaBlock(&mut storage_ix, (*s).storage_.slice_mut());
+    }
   }
   if let IsFirst::BothCatableBytesWritten = s.is_first_mb {
     // nothing to do here, move along