feat: support encode zero value in protobuf to adjust to go proto2 (#274

)

* feat: support encode zero value in protobuf to adjust to go proto2

* remove unused file

* chore: add test for more type and use a better feature name

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["pilota", "pilota-build", "pilota-thrift-parser"]
+members = ["pilota", "pilota-build", "pilota-thrift-parser", "examples"]
 resolver = "2"
 
 [profile.bench]

examples/Cargo.toml

@@ -0,0 +1,58 @@
+[package]
+name = "examples"
+version = "0.1.0"
+edition = "2021"
+description = "Compile thrift and protobuf idl into rust code at compile-time."
+homepage = "https://cloudwego.io/docs/pilota/"
+repository = "https://github.com/cloudwego/pilota"
+license = "MIT OR Apache-2.0"
+authors = ["Pilota Team <[email protected]>"]
+keywords = ["serialization", "thrift", "protobuf", "volo"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[badges]
+maintenance = { status = "actively-developed" }
+
+[dependencies]
+pilota = { path = "../pilota", features = ["pb-encode-default-value"] }
+pilota-build = { path = "../pilota-build" }
+pilota-thrift-parser = { path = "../pilota-thrift-parser", version = "0.11" }
+
+ahash = "0.8"
+anyhow = "1"
+dashmap = "5"
+heck = "0.5"
+itertools = "0.13"
+normpath = "1"
+paste = "1"
+petgraph = "0.6"
+phf = { version = "0.11", features = ["macros"] }
+proc-macro2 = "1"
+quote = "1"
+rayon = "1"
+rustc-hash = "1"
+salsa = { version = "0.17.0-pre.2" }
+scoped-tls = "1"
+serde = { version = "1", features = ["derive"] }
+serde_yaml = "0.9"
+syn = "2"
+toml = "0.8"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+# The official rust-protobuf parser currently has some bug.
+# We will switch to the official one when https://github.com/stepancheg/rust-protobuf/pull/646 is fixed.
+protobuf-parse = { package = "protobuf-parse2", version = "4.0.0-alpha.4" }
+protobuf = { package = "protobuf2", version = "4.0.0-alpha.2" }
+faststr = "0.2"
+
+[dev-dependencies]
+
+tokio = { version = "1", features = ["io-util"] }
+derivative = "2"
+tempfile = "3"
+diffy = "0.4"
+criterion = { version = "0.5", features = ["html_reports"] }
+rand = "0.8"
+linkedbytes = "0.1"

examples/idl/zero_value.proto

@@ -0,0 +1,5 @@
+message A {
+    map<string, string> str_map = 1;
+    required string s1 = 2;
+    optional string s2 = 3;
+}

examples/src/lib.rs

@@ -0,0 +1,33 @@
+use pilota::prost::Message as _;
+
+mod zero_value;
+
+#[test]
+fn test_pb_encode_zero_value() {
+    let test_data = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("idl")
+        .join("zero_value.proto");
+
+    let out_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("src")
+        .join("zero_value.rs");
+
+    pilota_build::Builder::protobuf()
+        .ignore_unused(false)
+        .include_dirs(vec![test_data.parent().unwrap().to_path_buf()])
+        .compile_with_config(
+            vec![pilota_build::IdlService::from_path(test_data.to_path_buf())],
+            pilota_build::Output::File(out_path.into()),
+        );
+
+    let mut a = zero_value::zero_value::A::default();
+
+    a.str_map.insert("key1".into(), "value".into());
+    a.str_map.insert("key2".into(), "".into());
+
+    let mut buf = pilota::BytesMut::new();
+    a.encode(&mut buf).unwrap();
+
+    println!("{:?}", buf);
+    println!("{:?}", buf.freeze().as_ref());
+}

examples/src/zero_value.rs

@@ -0,0 +1,102 @@
+pub mod zero_value {
+    #![allow(warnings, clippy::all)]
+    #[derive(Debug, Default, Clone, PartialEq)]
+    pub struct A {
+        pub str_map: ::pilota::AHashMap<::pilota::FastStr, ::pilota::FastStr>,
+
+        pub s1: ::pilota::FastStr,
+
+        pub s2: ::std::option::Option<::pilota::FastStr>,
+    }
+    impl ::pilota::prost::Message for A {
+        #[inline]
+        fn encoded_len(&self) -> usize {
+            0 + ::pilota::prost::encoding::hash_map::encoded_len(
+                ::pilota::prost::encoding::faststr::encoded_len,
+                ::pilota::prost::encoding::faststr::encoded_len,
+                1,
+                &self.str_map,
+            ) + ::pilota::prost::encoding::faststr::encoded_len(2, &self.s1)
+                + self.s2.as_ref().map_or(0, |value| {
+                    ::pilota::prost::encoding::faststr::encoded_len(3, value)
+                })
+        }
+
+        #[allow(unused_variables)]
+        fn encode_raw<B>(&self, buf: &mut B)
+        where
+            B: ::pilota::prost::bytes::BufMut,
+        {
+            ::pilota::prost::encoding::hash_map::encode(
+                ::pilota::prost::encoding::faststr::encode,
+                ::pilota::prost::encoding::faststr::encoded_len,
+                ::pilota::prost::encoding::faststr::encode,
+                ::pilota::prost::encoding::faststr::encoded_len,
+                1,
+                &self.str_map,
+                buf,
+            );
+            ::pilota::prost::encoding::faststr::encode(2, &self.s1, buf);
+            if let Some(_pilota_inner_value) = self.s2.as_ref() {
+                ::pilota::prost::encoding::faststr::encode(3, _pilota_inner_value, buf);
+            };
+        }
+
+        #[allow(unused_variables)]
+        fn merge_field<B>(
+            &mut self,
+            tag: u32,
+            wire_type: ::pilota::prost::encoding::WireType,
+            buf: &mut B,
+            ctx: ::pilota::prost::encoding::DecodeContext,
+        ) -> ::core::result::Result<(), ::pilota::prost::DecodeError>
+        where
+            B: ::pilota::prost::bytes::Buf,
+        {
+            const STRUCT_NAME: &'static str = stringify!(A);
+            match tag {
+                1 => {
+                    let mut _inner_pilota_value = &mut self.str_map;
+                    ::pilota::prost::encoding::hash_map::merge(
+                        ::pilota::prost::encoding::faststr::merge,
+                        ::pilota::prost::encoding::faststr::merge,
+                        &mut _inner_pilota_value,
+                        buf,
+                        ctx,
+                    )
+                    .map_err(|mut error| {
+                        error.push(STRUCT_NAME, stringify!(str_map));
+                        error
+                    })
+                }
+                2 => {
+                    let mut _inner_pilota_value = &mut self.s1;
+                    ::pilota::prost::encoding::faststr::merge(
+                        wire_type,
+                        _inner_pilota_value,
+                        buf,
+                        ctx,
+                    )
+                    .map_err(|mut error| {
+                        error.push(STRUCT_NAME, stringify!(s1));
+                        error
+                    })
+                }
+                3 => {
+                    let mut _inner_pilota_value = &mut self.s2;
+                    ::pilota::prost::encoding::faststr::merge(
+                        wire_type,
+                        _inner_pilota_value.get_or_insert_with(::core::default::Default::default),
+                        buf,
+                        ctx,
+                    )
+                    .map_err(|mut error| {
+                        error.push(STRUCT_NAME, stringify!(s2));
+                        error
+                    })
+                }
+                _ => ::pilota::prost::encoding::skip_field(wire_type, tag, buf, ctx),
+            }
+        }
+    }
+}

pilota/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pilota"
-version = "0.11.3"
+version = "0.11.4"
 edition = "2021"
 description = "Pilota is a thrift and protobuf implementation in pure rust with high performance and extensibility."
 documentation = "https://docs.rs/pilota"
@@ -41,6 +41,7 @@ rand = "0.8"
 
 [features]
 unstable = []
+pb-encode-default-value = []
 
 [[bench]]
 name = "faststr"

pilota/src/prost/encoding.rs

@@ -1579,8 +1579,14 @@ macro_rules! map {
             VL: Fn(u32, &V) -> usize,
         {
             for (key, val) in values.iter() {
-                let skip_key = key == &K::default();
-                let skip_val = val == val_default;
+                let mut skip_key = key == &K::default();
+                let mut skip_val = val == val_default;
+
+                #[cfg(feature = "pb-encode-zero-value")]
+                {
+                    skip_key = false;
+                    skip_val = false;
+                }
 
                 let len = (if skip_key { 0 } else { key_encoded_len(1, key) })
                     + (if skip_val { 0 } else { val_encoded_len(2, val) });