From d561e05b6d914d637f0cd5156737f6f610415307 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 31 Mar 2022 18:34:20 -0700 Subject: [PATCH 1/7] remove attributes that make rust generated files unsuported with include!(...) --- serde-generate/src/rust.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/serde-generate/src/rust.rs b/serde-generate/src/rust.rs index ff5cda30d..c2a96aa0b 100644 --- a/serde-generate/src/rust.rs +++ b/serde-generate/src/rust.rs @@ -186,10 +186,6 @@ where .flatten() .cloned() .collect::>(); - writeln!(self.out, "#![allow(unused_imports)]")?; - if !external_names.contains("Map") { - writeln!(self.out, "use std::collections::BTreeMap as Map;")?; - } if self.generator.config.serialization { writeln!(self.out, "use serde::{{Serialize, Deserialize}};")?; } From 57b1f49dea68b080748630b14083a0feb92adcc8 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 20 Jul 2022 11:45:05 -0700 Subject: [PATCH 2/7] [generate] add debugging output for node --- serde-generate/src/analyzer.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/serde-generate/src/analyzer.rs b/serde-generate/src/analyzer.rs index 52ec542b7..250ca1dc9 100644 --- a/serde-generate/src/analyzer.rs +++ b/serde-generate/src/analyzer.rs @@ -45,7 +45,7 @@ pub fn get_dependency_map_with_external_dependencies<'a>( /// Classic topological sorting algorithm except that it doesn't abort in case of cycles. pub fn best_effort_topological_sort(children: &BTreeMap>) -> Vec where - T: Clone + std::cmp::Ord + std::cmp::Eq + std::hash::Hash, + T: Clone + std::cmp::Eq + std::cmp::Ord + std::fmt::Debug + std::hash::Hash, { // Build the initial queue so that we pick up nodes with less children first (and otherwise // those with smaller key first). @@ -81,7 +81,12 @@ where // 2. Schedule all the (yet unseen) children then this node for a second visit. // (If possible, visit children by increasing key.) queue.push(node.clone()); - for child in children[&node].iter().rev() { + for child in children + .get(&node) + .unwrap_or_else(|| panic!("Unable to find node: {:?}", node)) + .iter() + .rev() + { if !seen.contains(child) { queue.push(child.clone()); } From 8d61d6a1911d0a26982f31a33f2ed397283433a8 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 22 Jul 2022 15:19:13 -0700 Subject: [PATCH 3/7] fix golang support --- serde-generate/runtime/golang/bcs/bcs_test.go | 4 ++-- serde-generate/runtime/golang/bcs/deserializer.go | 2 +- serde-generate/runtime/golang/bcs/serializer.go | 2 +- serde-generate/runtime/golang/bincode/deserializer.go | 2 +- serde-generate/runtime/golang/bincode/serializer.go | 2 +- serde-generate/runtime/golang/go.mod | 2 +- serde-generate/src/golang.rs | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/serde-generate/runtime/golang/bcs/bcs_test.go b/serde-generate/runtime/golang/bcs/bcs_test.go index 2c23c540b..7345d6e5c 100644 --- a/serde-generate/runtime/golang/bcs/bcs_test.go +++ b/serde-generate/runtime/golang/bcs/bcs_test.go @@ -7,8 +7,8 @@ import ( "fmt" "testing" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/bcs" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang/bcs" + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang/serde" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/serde-generate/runtime/golang/bcs/deserializer.go b/serde-generate/runtime/golang/bcs/deserializer.go index fc03ba808..36a16e979 100644 --- a/serde-generate/runtime/golang/bcs/deserializer.go +++ b/serde-generate/runtime/golang/bcs/deserializer.go @@ -7,7 +7,7 @@ import ( "bytes" "errors" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" ) // Maximum length allowed for sequences (vectors, bytes, strings) and maps. diff --git a/serde-generate/runtime/golang/bcs/serializer.go b/serde-generate/runtime/golang/bcs/serializer.go index 712f1efc9..4f421bd21 100644 --- a/serde-generate/runtime/golang/bcs/serializer.go +++ b/serde-generate/runtime/golang/bcs/serializer.go @@ -8,7 +8,7 @@ import ( "errors" "sort" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" ) // `serializer` extends `serde.BinarySerializer` to implement `serde.Serializer`. diff --git a/serde-generate/runtime/golang/bincode/deserializer.go b/serde-generate/runtime/golang/bincode/deserializer.go index e1701cce6..79ece2222 100644 --- a/serde-generate/runtime/golang/bincode/deserializer.go +++ b/serde-generate/runtime/golang/bincode/deserializer.go @@ -7,7 +7,7 @@ import ( "errors" "math" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" ) // MaxSequenceLength is max length supported in practice (e.g. in Java). diff --git a/serde-generate/runtime/golang/bincode/serializer.go b/serde-generate/runtime/golang/bincode/serializer.go index d104e4c32..cae22f560 100644 --- a/serde-generate/runtime/golang/bincode/serializer.go +++ b/serde-generate/runtime/golang/bincode/serializer.go @@ -6,7 +6,7 @@ package bincode import ( "math" - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" ) // `serializer` extends `serde.BinarySerializer` to implement `serde.Serializer`. diff --git a/serde-generate/runtime/golang/go.mod b/serde-generate/runtime/golang/go.mod index be71bea9e..5a20b866c 100644 --- a/serde-generate/runtime/golang/go.mod +++ b/serde-generate/runtime/golang/go.mod @@ -1,4 +1,4 @@ -module github.com/novifinancial/serde-reflection/serde-generate/runtime/golang +module github.com/zefchain/serde-reflection/serde-generate/runtime/golang go 1.14 diff --git a/serde-generate/src/golang.rs b/serde-generate/src/golang.rs index 1a9128b16..0bc10d6e1 100644 --- a/serde-generate/src/golang.rs +++ b/serde-generate/src/golang.rs @@ -19,7 +19,7 @@ pub struct CodeGenerator<'a> { /// Language-independent configuration. config: &'a CodeGeneratorConfig, /// Module path where to find the serde runtime packages (serde, bcs, bincode). - /// Default: "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang". + /// Default: "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang". serde_module_path: String, /// Mapping from external type names to fully-qualified class names (e.g. "MyClass" -> "com.my_org.my_package.MyClass"). /// Derived from `config.external_definitions`. @@ -60,7 +60,7 @@ impl<'a> CodeGenerator<'a> { Self { config, serde_module_path: - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang" + "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang" .to_string(), external_qualified_names, } From 9effb0d22723fb2703284ef6d8c27e831cef66b0 Mon Sep 17 00:00:00 2001 From: Greg Nazario Date: Tue, 6 Sep 2022 22:42:36 -0700 Subject: [PATCH 4/7] Merge pull request #2 from andrewl33/add-vecbytes Add support for nested byte vectors --- serde-generate/runtime/golang/bcs/deserializer.go | 2 +- serde-generate/runtime/golang/bcs/serializer.go | 2 +- serde-generate/runtime/golang/bincode/deserializer.go | 2 +- serde-generate/runtime/golang/bincode/serializer.go | 2 +- serde-generate/runtime/python/serde_binary/__init__.py | 4 ++-- serde-generate/src/golang.rs | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/serde-generate/runtime/golang/bcs/deserializer.go b/serde-generate/runtime/golang/bcs/deserializer.go index 36a16e979..2790500b3 100644 --- a/serde-generate/runtime/golang/bcs/deserializer.go +++ b/serde-generate/runtime/golang/bcs/deserializer.go @@ -7,7 +7,7 @@ import ( "bytes" "errors" - "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang/serde" ) // Maximum length allowed for sequences (vectors, bytes, strings) and maps. diff --git a/serde-generate/runtime/golang/bcs/serializer.go b/serde-generate/runtime/golang/bcs/serializer.go index 4f421bd21..8d789b090 100644 --- a/serde-generate/runtime/golang/bcs/serializer.go +++ b/serde-generate/runtime/golang/bcs/serializer.go @@ -8,7 +8,7 @@ import ( "errors" "sort" - "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang/serde" ) // `serializer` extends `serde.BinarySerializer` to implement `serde.Serializer`. diff --git a/serde-generate/runtime/golang/bincode/deserializer.go b/serde-generate/runtime/golang/bincode/deserializer.go index 79ece2222..21d475701 100644 --- a/serde-generate/runtime/golang/bincode/deserializer.go +++ b/serde-generate/runtime/golang/bincode/deserializer.go @@ -7,7 +7,7 @@ import ( "errors" "math" - "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang/serde" ) // MaxSequenceLength is max length supported in practice (e.g. in Java). diff --git a/serde-generate/runtime/golang/bincode/serializer.go b/serde-generate/runtime/golang/bincode/serializer.go index cae22f560..04f62b20e 100644 --- a/serde-generate/runtime/golang/bincode/serializer.go +++ b/serde-generate/runtime/golang/bincode/serializer.go @@ -6,7 +6,7 @@ package bincode import ( "math" - "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang/serde" + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang/serde" ) // `serializer` extends `serde.BinarySerializer` to implement `serde.Serializer`. diff --git a/serde-generate/runtime/python/serde_binary/__init__.py b/serde-generate/runtime/python/serde_binary/__init__.py index 2a02e98bb..4bb36b23f 100644 --- a/serde-generate/runtime/python/serde_binary/__init__.py +++ b/serde-generate/runtime/python/serde_binary/__init__.py @@ -142,7 +142,7 @@ def serialize_any(self, obj: typing.Any, obj_type): self.serialize_any(item, item_type) elif getattr(obj_type, "__origin__") == tuple: # Tuple - if len(types) != 1 or types[0] != (): + if len(types) is not 1 or types[0] is not (): for i in range(len(obj)): self.serialize_any(obj[i], types[i]) @@ -348,7 +348,7 @@ def deserialize_any(self, obj_type) -> typing.Any: elif getattr(obj_type, "__origin__") == tuple: # Tuple result = [] - if len(types) == 1 and types[0] == (): + if len(types) is 1 and types[0] is (): return tuple() for i in range(len(types)): item = self.deserialize_any(types[i]) diff --git a/serde-generate/src/golang.rs b/serde-generate/src/golang.rs index 0bc10d6e1..811a59bbe 100644 --- a/serde-generate/src/golang.rs +++ b/serde-generate/src/golang.rs @@ -19,7 +19,7 @@ pub struct CodeGenerator<'a> { /// Language-independent configuration. config: &'a CodeGeneratorConfig, /// Module path where to find the serde runtime packages (serde, bcs, bincode). - /// Default: "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang". + /// Default: "github.com/zefchain/serde-reflection/serde-generate/runtime/golang". serde_module_path: String, /// Mapping from external type names to fully-qualified class names (e.g. "MyClass" -> "com.my_org.my_package.MyClass"). /// Derived from `config.external_definitions`. @@ -60,7 +60,7 @@ impl<'a> CodeGenerator<'a> { Self { config, serde_module_path: - "github.com/aptos-labs/serde-reflection/serde-generate/runtime/golang" + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang" .to_string(), external_qualified_names, } From 311787e1c2bbf9f43b84c74fa0353597728375e9 Mon Sep 17 00:00:00 2001 From: Tim Zakian Date: Wed, 18 Aug 2021 14:59:40 -0700 Subject: [PATCH 5/7] [serde-generate] Add support for nested byte vectors --- .../csharp/Serde/BinaryDeserializer.cs | 16 ++++++++ .../runtime/csharp/Serde/BinarySerializer.cs | 7 ++++ .../runtime/csharp/Serde/IDeserializer.cs | 2 + .../runtime/csharp/Serde/ISerializer.cs | 2 + serde-generate/runtime/golang/bcs/bcs_test.go | 41 +++++++++++++++++++ .../runtime/golang/bcs/deserializer.go | 4 ++ .../runtime/golang/bcs/serializer.go | 4 ++ .../runtime/golang/bincode/deserializer.go | 4 ++ .../runtime/golang/bincode/serializer.go | 4 ++ .../golang/serde/binary_deserializer.go | 17 ++++++++ .../runtime/golang/serde/binary_serializer.go | 8 ++++ .../runtime/golang/serde/interfaces.go | 4 ++ .../com/novi/serde/BinaryDeserializer.java | 15 +++++++ .../java/com/novi/serde/BinarySerializer.java | 8 ++++ .../java/com/novi/serde/Deserializer.java | 3 ++ .../java/com/novi/serde/Serializer.java | 3 ++ serde-generate/runtime/python/bcs/test_bcs.py | 22 ++++++++++ .../runtime/python/serde_binary/__init__.py | 12 ++++++ 18 files changed, 176 insertions(+) diff --git a/serde-generate/runtime/csharp/Serde/BinaryDeserializer.cs b/serde-generate/runtime/csharp/Serde/BinaryDeserializer.cs index b2e370e6b..5a5f2861c 100644 --- a/serde-generate/runtime/csharp/Serde/BinaryDeserializer.cs +++ b/serde-generate/runtime/csharp/Serde/BinaryDeserializer.cs @@ -65,6 +65,22 @@ public string deserialize_str() return utf8.GetString(content); } + public ValueArray> deserialize_vec_bytes() { + long len = deserialize_len(); + ValueArray[] content = new ValueArray[len]; + + if (len < 0 || len > int.MaxValue) + { + throw new DeserializationException("Incorrect length value for C# array"); + } + + for (long i = 0; i < len; i++) { + content[i] = deserialize_bytes(); + } + + return new ValueArray>(content); + } + public ValueArray deserialize_bytes() { long len = deserialize_len(); diff --git a/serde-generate/runtime/csharp/Serde/BinarySerializer.cs b/serde-generate/runtime/csharp/Serde/BinarySerializer.cs index 67a739b7e..64d5a1d94 100644 --- a/serde-generate/runtime/csharp/Serde/BinarySerializer.cs +++ b/serde-generate/runtime/csharp/Serde/BinarySerializer.cs @@ -64,6 +64,13 @@ public void decrease_container_depth() public void serialize_str(string value) => serialize_bytes(new ValueArray(utf8.GetBytes(value))); + public void serialize_vec_bytes(ValueArray> value) { + serialize_len(value.Count); + foreach (ValueArray v in value) { + serialize_bytes(v); + } + } + public void serialize_bytes(ValueArray value) { serialize_len(value.Count); diff --git a/serde-generate/runtime/csharp/Serde/IDeserializer.cs b/serde-generate/runtime/csharp/Serde/IDeserializer.cs index fc469d31f..fd487d5aa 100644 --- a/serde-generate/runtime/csharp/Serde/IDeserializer.cs +++ b/serde-generate/runtime/csharp/Serde/IDeserializer.cs @@ -9,6 +9,8 @@ public interface IDeserializer { string deserialize_str(); + ValueArray> deserialize_vec_bytes(); + ValueArray deserialize_bytes(); bool deserialize_bool(); diff --git a/serde-generate/runtime/csharp/Serde/ISerializer.cs b/serde-generate/runtime/csharp/Serde/ISerializer.cs index 0bc59ff2a..41f4add2b 100644 --- a/serde-generate/runtime/csharp/Serde/ISerializer.cs +++ b/serde-generate/runtime/csharp/Serde/ISerializer.cs @@ -9,6 +9,8 @@ public interface ISerializer { void serialize_str(string value); + void serialize_vec_bytes(ValueArray> value); + void serialize_bytes(ValueArray value); void serialize_bool(bool value); diff --git a/serde-generate/runtime/golang/bcs/bcs_test.go b/serde-generate/runtime/golang/bcs/bcs_test.go index 7345d6e5c..06ff150b2 100644 --- a/serde-generate/runtime/golang/bcs/bcs_test.go +++ b/serde-generate/runtime/golang/bcs/bcs_test.go @@ -50,6 +50,47 @@ func TestSerializeDeserializeBytes(t *testing.T) { }) } +func TestSerializeDeserializeVecBytes(t *testing.T) { + cases := []struct { + target [][]byte + expected []byte + }{ + { + target: [][]byte{{1, 2, 38}, {0, 1}, {0}}, + expected: []byte{3, 3, 1, 2, 38, 2, 0, 1, 1, 0}, + }, + { + target: [][]byte{{1, 2, 38}, {0, 1}, {}}, + expected: []byte{3, 3, 1, 2, 38, 2, 0, 1, 0}, + }, + { + target: [][]byte{}, + expected: []byte{0}, + }, + } + + for _, tc := range cases { + t.Run(fmt.Sprintf("%#v", tc.target), func(t *testing.T) { + s := bcs.NewSerializer() + d := bcs.NewDeserializer(tc.expected) + + err := s.SerializeVecBytes(tc.target) + require.NoError(t, err) + + deserialized, err := d.DeserializeVecBytes() + require.NoError(t, err) + + assert.Equal(t, tc.expected, s.GetBytes()) + assert.Equal(t, tc.target, deserialized) + }) + } + t.Run("deserialize error: EOF", func(t *testing.T) { + d := bcs.NewDeserializer([]byte{}) + _, err := d.DeserializeVecBytes() + require.EqualError(t, err, "EOF") + }) +} + func TestSerializeDeserializeStr(t *testing.T) { cases := []struct { target string diff --git a/serde-generate/runtime/golang/bcs/deserializer.go b/serde-generate/runtime/golang/bcs/deserializer.go index 2790500b3..e40e94fa6 100644 --- a/serde-generate/runtime/golang/bcs/deserializer.go +++ b/serde-generate/runtime/golang/bcs/deserializer.go @@ -37,6 +37,10 @@ func (d *deserializer) DeserializeF64() (float64, error) { return 0, errors.New("unimplemented") } +func (d *deserializer) DeserializeVecBytes() ([][]byte, error) { + return d.BinaryDeserializer.DeserializeVecBytes(d.DeserializeLen) +} + func (d *deserializer) DeserializeBytes() ([]byte, error) { return d.BinaryDeserializer.DeserializeBytes(d.DeserializeLen) } diff --git a/serde-generate/runtime/golang/bcs/serializer.go b/serde-generate/runtime/golang/bcs/serializer.go index 8d789b090..6b6c90b7e 100644 --- a/serde-generate/runtime/golang/bcs/serializer.go +++ b/serde-generate/runtime/golang/bcs/serializer.go @@ -34,6 +34,10 @@ func (s *serializer) SerializeStr(value string) error { return s.BinarySerializer.SerializeStr(value, s.SerializeLen) } +func (s *serializer) SerializeVecBytes(value [][]byte) error { + return s.BinarySerializer.SerializeVecBytes(value, s.SerializeLen) +} + func (s *serializer) SerializeBytes(value []byte) error { return s.BinarySerializer.SerializeBytes(value, s.SerializeLen) } diff --git a/serde-generate/runtime/golang/bincode/deserializer.go b/serde-generate/runtime/golang/bincode/deserializer.go index 21d475701..16363d533 100644 --- a/serde-generate/runtime/golang/bincode/deserializer.go +++ b/serde-generate/runtime/golang/bincode/deserializer.go @@ -32,6 +32,10 @@ func (d *deserializer) DeserializeF64() (float64, error) { return math.Float64frombits(ret), err } +func (d *deserializer) DeserializeVecBytes() ([][]byte, error) { + return d.BinaryDeserializer.DeserializeVecBytes(d.DeserializeLen) +} + func (d *deserializer) DeserializeBytes() ([]byte, error) { return d.BinaryDeserializer.DeserializeBytes(d.DeserializeLen) } diff --git a/serde-generate/runtime/golang/bincode/serializer.go b/serde-generate/runtime/golang/bincode/serializer.go index 04f62b20e..8aee9b843 100644 --- a/serde-generate/runtime/golang/bincode/serializer.go +++ b/serde-generate/runtime/golang/bincode/serializer.go @@ -30,6 +30,10 @@ func (s *serializer) SerializeStr(value string) error { return s.BinarySerializer.SerializeStr(value, s.SerializeLen) } +func (s *serializer) SerializeVecBytes(value [][]byte) error { + return s.BinarySerializer.SerializeVecBytes(value, s.SerializeLen) +} + func (s *serializer) SerializeBytes(value []byte) error { return s.BinarySerializer.SerializeBytes(value, s.SerializeLen) } diff --git a/serde-generate/runtime/golang/serde/binary_deserializer.go b/serde-generate/runtime/golang/serde/binary_deserializer.go index 8043552ab..e405047bc 100644 --- a/serde-generate/runtime/golang/serde/binary_deserializer.go +++ b/serde-generate/runtime/golang/serde/binary_deserializer.go @@ -38,6 +38,23 @@ func (d *BinaryDeserializer) DecreaseContainerDepth() { d.containerDepthBudget += 1 } +func (d *BinaryDeserializer) DeserializeVecBytes(deserializeLen func() (uint64, error)) ([][]byte, error) { + len, err := deserializeLen() + if err != nil { + return nil, err + } + ret := make([][]byte, len) + for i := 0; uint64(i) < len; i++ { + bytes, err := d.DeserializeBytes(deserializeLen) + if err != nil { + return nil, err + } + ret[i] = bytes + } + + return ret, err +} + // `deserializeLen` to be provided by the extending struct. func (d *BinaryDeserializer) DeserializeBytes(deserializeLen func() (uint64, error)) ([]byte, error) { len, err := deserializeLen() diff --git a/serde-generate/runtime/golang/serde/binary_serializer.go b/serde-generate/runtime/golang/serde/binary_serializer.go index 6b1d7eb8a..075eacbca 100644 --- a/serde-generate/runtime/golang/serde/binary_serializer.go +++ b/serde-generate/runtime/golang/serde/binary_serializer.go @@ -33,6 +33,14 @@ func (d *BinarySerializer) DecreaseContainerDepth() { d.containerDepthBudget += 1 } +func (s *BinarySerializer) SerializeVecBytes(value [][]byte, serializeLen func(uint64) error) error { + serializeLen(uint64(len(value))) + for _, bytes := range value { + s.SerializeBytes(bytes, serializeLen) + } + return nil +} + // `serializeLen` to be provided by the extending struct. func (s *BinarySerializer) SerializeBytes(value []byte, serializeLen func(uint64) error) error { serializeLen(uint64(len(value))) diff --git a/serde-generate/runtime/golang/serde/interfaces.go b/serde-generate/runtime/golang/serde/interfaces.go index e4eaba549..6f86cbdc5 100644 --- a/serde-generate/runtime/golang/serde/interfaces.go +++ b/serde-generate/runtime/golang/serde/interfaces.go @@ -6,6 +6,8 @@ package serde type Serializer interface { SerializeStr(value string) error + SerializeVecBytes(value [][]byte) error + SerializeBytes(value []byte) error SerializeBool(value bool) error @@ -58,6 +60,8 @@ type Serializer interface { type Deserializer interface { DeserializeStr() (string, error) + DeserializeVecBytes() ([][]byte, error) + DeserializeBytes() ([]byte, error) DeserializeBool() (bool, error) diff --git a/serde-generate/runtime/java/com/novi/serde/BinaryDeserializer.java b/serde-generate/runtime/java/com/novi/serde/BinaryDeserializer.java index 94c56e455..b0acae7d4 100644 --- a/serde-generate/runtime/java/com/novi/serde/BinaryDeserializer.java +++ b/serde-generate/runtime/java/com/novi/serde/BinaryDeserializer.java @@ -9,6 +9,8 @@ import java.nio.charset.StandardCharsets; import java.nio.charset.CharacterCodingException; import java.math.BigInteger; +import java.util.List; +import java.util.ArrayList; public abstract class BinaryDeserializer implements Deserializer { protected ByteBuffer input; @@ -47,6 +49,19 @@ public String deserialize_str() throws DeserializationError { return new String(content); } + public List deserialize_vec_bytes() throws DeserializationError { + long len = deserialize_len(); + if (len < 0 || len > Integer.MAX_VALUE) { + throw new DeserializationError("Incorrect length value for Java array"); + } + List content = new ArrayList(); + for (int i = 0; i < len; i++) { + content.add(deserialize_bytes()); + } + + return content; + } + public Bytes deserialize_bytes() throws DeserializationError { long len = deserialize_len(); if (len < 0 || len > Integer.MAX_VALUE) { diff --git a/serde-generate/runtime/java/com/novi/serde/BinarySerializer.java b/serde-generate/runtime/java/com/novi/serde/BinarySerializer.java index bb0c5866c..af2bfce9d 100644 --- a/serde-generate/runtime/java/com/novi/serde/BinarySerializer.java +++ b/serde-generate/runtime/java/com/novi/serde/BinarySerializer.java @@ -4,6 +4,7 @@ package com.novi.serde; import java.math.BigInteger; +import java.util.List; public abstract class BinarySerializer implements Serializer { protected MyByteArrayOutputStream output; @@ -29,6 +30,13 @@ public void serialize_str(String value) throws SerializationError { serialize_bytes(new Bytes(value.getBytes())); } + public void serialize_vec_bytes(List value) throws SerializationError { + serialize_len(value.size()); + for (Bytes bytes : value) { + serialize_bytes(bytes); + } + } + public void serialize_bytes(Bytes value) throws SerializationError { byte[] content = value.content(); serialize_len(content.length); diff --git a/serde-generate/runtime/java/com/novi/serde/Deserializer.java b/serde-generate/runtime/java/com/novi/serde/Deserializer.java index 78f3940da..e5ed565b4 100644 --- a/serde-generate/runtime/java/com/novi/serde/Deserializer.java +++ b/serde-generate/runtime/java/com/novi/serde/Deserializer.java @@ -4,10 +4,13 @@ package com.novi.serde; import java.math.BigInteger; +import java.util.List; public interface Deserializer { String deserialize_str() throws DeserializationError; + List deserialize_vec_bytes() throws DeserializationError; + Bytes deserialize_bytes() throws DeserializationError; Boolean deserialize_bool() throws DeserializationError; diff --git a/serde-generate/runtime/java/com/novi/serde/Serializer.java b/serde-generate/runtime/java/com/novi/serde/Serializer.java index 266021fda..c44417a5f 100644 --- a/serde-generate/runtime/java/com/novi/serde/Serializer.java +++ b/serde-generate/runtime/java/com/novi/serde/Serializer.java @@ -4,10 +4,13 @@ package com.novi.serde; import java.math.BigInteger; +import java.util.List; public interface Serializer { void serialize_str(String value) throws SerializationError; + void serialize_vec_bytes(List value) throws SerializationError; + void serialize_bytes(Bytes value) throws SerializationError; void serialize_bool(Boolean value) throws SerializationError; diff --git a/serde-generate/runtime/python/bcs/test_bcs.py b/serde-generate/runtime/python/bcs/test_bcs.py index 4155d454b..421f911f3 100644 --- a/serde-generate/runtime/python/bcs/test_bcs.py +++ b/serde-generate/runtime/python/bcs/test_bcs.py @@ -141,6 +141,28 @@ def test_serialize_bytes(self): self.assertEqual(bcs.deserialize(b"\x00", bytes), (b"", b"")) + def test_serialize_vec_bytes(self): + a = [] + b = [b"\x00\x00", b"\x01\x01"] + c = [b"\x00" * 128, b"\x01" * 127, b"\x02" * 3, b""] + + aser = bcs.serialize(a, typing.Sequence[bytes]) + bser = bcs.serialize(b, typing.Sequence[bytes]) + cser = bcs.serialize(c, typing.Sequence[bytes]) + + self.assertEqual(aser, b"\x00") + self.assertEqual(bser, b"\x02\x02\x00\x00\x02\x01\x01") + self.assertEqual(cser, + b"\x04\x80\x01" + b"\x00" * 128 + + b"\x7F" + b"\x01" * 127 + + b"\x03" + b"\x02" * 3 + + b"\x00" + ) + + self.assertEqual(bcs.deserialize(aser, typing.Sequence[bytes]), (a, b"")) + self.assertEqual(bcs.deserialize(bser, typing.Sequence[bytes]), (b, b"")) + self.assertEqual(bcs.deserialize(cser, typing.Sequence[bytes]), (c, b"")) + def test_serialize_tuple(self): T = typing.Tuple[st.uint8, st.uint16] self.assertEqual(bcs.serialize((0, 1), T), b"\x00\x01\x00") diff --git a/serde-generate/runtime/python/serde_binary/__init__.py b/serde-generate/runtime/python/serde_binary/__init__.py index 4bb36b23f..2b9ca293d 100644 --- a/serde-generate/runtime/python/serde_binary/__init__.py +++ b/serde-generate/runtime/python/serde_binary/__init__.py @@ -47,8 +47,15 @@ def __post_init__(self): st.char: self.serialize_char, str: self.serialize_str, bytes: self.serialize_bytes, + typing.Sequence[bytes]: self.serialize_vec_bytes, } + def serialize_vec_bytes(self, value: typing.Sequence[bytes]): + self.serialize_len(len(value)) + for byte_vec in value: + self.serialize_bytes(byte_vec) + + def serialize_bytes(self, value: bytes): self.serialize_len(len(value)) self.output.write(value) @@ -227,6 +234,7 @@ def __post_init__(self): st.char: self.deserialize_char, str: self.deserialize_str, bytes: self.deserialize_bytes, + typing.Sequence[bytes]: self.deserialize_vec_bytes, } def read(self, length: int) -> bytes: @@ -235,6 +243,10 @@ def read(self, length: int) -> bytes: raise st.DeserializationError("Input is too short") return value + def deserialize_vec_bytes(self) -> typing.Sequence[bytes]: + length = self.deserialize_len() + return [self.deserialize_bytes() for _ in range(length)] + def deserialize_bytes(self) -> bytes: length = self.deserialize_len() return self.read(length) From 098a417952b9409d6373340ee02bd24668ae11a0 Mon Sep 17 00:00:00 2001 From: Gerardo Di Giacomo <19227040+gedigi@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:33:38 -0700 Subject: [PATCH 6/7] cargo fmt --- serde-generate/src/golang.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/serde-generate/src/golang.rs b/serde-generate/src/golang.rs index 811a59bbe..6f3e3cac2 100644 --- a/serde-generate/src/golang.rs +++ b/serde-generate/src/golang.rs @@ -59,9 +59,8 @@ impl<'a> CodeGenerator<'a> { } Self { config, - serde_module_path: - "github.com/zefchain/serde-reflection/serde-generate/runtime/golang" - .to_string(), + serde_module_path: "github.com/zefchain/serde-reflection/serde-generate/runtime/golang" + .to_string(), external_qualified_names, } } From d7805ff25044568a42b9c977243a9695c7c964fc Mon Sep 17 00:00:00 2001 From: Gerardo Di Giacomo <19227040+gedigi@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:39:48 -0700 Subject: [PATCH 7/7] fix python code and golang modules/tests --- serde-generate/runtime/golang/go.sum | 1 + serde-generate/runtime/python/serde_binary/__init__.py | 4 ++-- serde-generate/tests/golang_generation.rs | 2 +- serde-generate/tests/golang_runtime.rs | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/serde-generate/runtime/golang/go.sum b/serde-generate/runtime/golang/go.sum index 56d62e7c2..afe7890c9 100644 --- a/serde-generate/runtime/golang/go.sum +++ b/serde-generate/runtime/golang/go.sum @@ -5,6 +5,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/serde-generate/runtime/python/serde_binary/__init__.py b/serde-generate/runtime/python/serde_binary/__init__.py index 2b9ca293d..92a7e7ce2 100644 --- a/serde-generate/runtime/python/serde_binary/__init__.py +++ b/serde-generate/runtime/python/serde_binary/__init__.py @@ -149,7 +149,7 @@ def serialize_any(self, obj: typing.Any, obj_type): self.serialize_any(item, item_type) elif getattr(obj_type, "__origin__") == tuple: # Tuple - if len(types) is not 1 or types[0] is not (): + if len(types) != 1 or types[0] != (): for i in range(len(obj)): self.serialize_any(obj[i], types[i]) @@ -360,7 +360,7 @@ def deserialize_any(self, obj_type) -> typing.Any: elif getattr(obj_type, "__origin__") == tuple: # Tuple result = [] - if len(types) is 1 and types[0] is (): + if len(types) == 1 and types[0] == (): return tuple() for i in range(len(types)): item = self.deserialize_any(types[i]) diff --git a/serde-generate/tests/golang_generation.rs b/serde-generate/tests/golang_generation.rs index ef30ef5ce..65843c5e6 100644 --- a/serde-generate/tests/golang_generation.rs +++ b/serde-generate/tests/golang_generation.rs @@ -69,7 +69,7 @@ fn test_that_golang_code_compiles_with_config_and_registry( .arg("edit") .arg("-replace") .arg(format!( - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang={}", + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang={}", runtime_mod_path.to_str().unwrap() )) .status() diff --git a/serde-generate/tests/golang_runtime.rs b/serde-generate/tests/golang_runtime.rs index 506cd9d02..44fd798c1 100644 --- a/serde-generate/tests/golang_runtime.rs +++ b/serde-generate/tests/golang_runtime.rs @@ -110,7 +110,7 @@ func main() {{ .arg("edit") .arg("-replace") .arg(format!( - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang={}", + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang={}", runtime_mod_path.to_str().unwrap() )) .status() @@ -241,7 +241,7 @@ func main() {{ .arg("edit") .arg("-replace") .arg(format!( - "github.com/novifinancial/serde-reflection/serde-generate/runtime/golang={}", + "github.com/zefchain/serde-reflection/serde-generate/runtime/golang={}", runtime_mod_path.to_str().unwrap() )) .status()