Skip to content
This repository was archived by the owner on Feb 8, 2022. It is now read-only.

Commit

Permalink
Add proto3 specific deserialization that requires fewer SRTP constrai…
Browse files Browse the repository at this point in the history 
…nts (#85)

* Add proto3 specific deserialization that requires fewer SRTP constraints
* Refactor to share as much as possible between proto2/3 deserialization
* Fix tests to refere to proto2 on deserialize
* Removed UnknownFields from the xmldoc
* A property named UnknownFields is not required by SRTP constraints
* Use the 0 field mapping for unknown fields if present, no action if this is missing.
    - This gives the code generator the choice of generating the 0 field maping to a property, therefore adhering to the proto3.5 chanages to make implementation preserve unknown fields on deserialisation.
* Add unit tests for proto3
* Remove unecessary RequiredFields
* Add some documentation about proto3 definitions
* preserve any previous unknown fields on re-serialization
  • Loading branch information
@7sharp9 @jhugard
7sharp9 authored and jhugard committed Apr 18, 2018
1 parent bd78b5a commit ca65a0f
Show file tree
Hide file tree
Showing 5 changed files with 424 additions and 150 deletions.
16 changes: 8 additions & 8 deletions Serialization.Test/ExampleProtoClass.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,13 @@ module SampleNamespace =
member m.Serialize zcb = Serialize.toZeroCopyBuffer m zcb
member m.SerializeLengthDelimited () = Serialize.toArrayLD m

static member Deserialize buf = buf |> Deserialize.fromArray (InnerMessage())
static member Deserialize buf = buf |> Deserialize.fromArraySegment (InnerMessage())
static member Deserialize zcb = zcb |> Deserialize.fromZeroCopyBuffer (InnerMessage())
static member Deserialize raw = raw |> Deserialize.fromRawField (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.fromArrayLD (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.fromArraySegmentLD (InnerMessage())
static member DeserializeLengthDelimited zcb = zcb |> Deserialize.fromZcbLD (InnerMessage())
static member Deserialize buf = buf |> Deserialize.Proto2.fromArray (InnerMessage())
static member Deserialize buf = buf |> Deserialize.Proto2.fromArraySegment (InnerMessage())
static member Deserialize zcb = zcb |> Deserialize.Proto2.fromZeroCopyBuffer (InnerMessage())
static member Deserialize raw = raw |> Deserialize.Proto2.fromRawField (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.Proto2.fromArrayLD (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.Proto2.fromArraySegmentLD (InnerMessage())
static member DeserializeLengthDelimited zcb = zcb |> Deserialize.Proto2.fromZcbLD (InnerMessage())



Expand Down Expand Up @@ -158,7 +158,7 @@ module PerformanceTest =
let zcr = Froto.Serialization.ZeroCopyBuffer(zcw.Array)
seq {
while not zcr.IsEof do
yield zcr |> Deserialize.fromZcbLD (InnerMessage())
yield zcr |> Deserialize.Proto2.fromZcbLD (InnerMessage())
}

ys |> Seq.iter ignore
Expand Down
2 changes: 1 addition & 1 deletion Serialization.Test/ExampleProtoRecord.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ module PerformanceTest =
let zcr = Froto.Serialization.ZeroCopyBuffer(zcw.Array)
seq {
while not zcr.IsEof do
yield zcr |> Deserialize.fromZcbLD InnerSample.Default
yield zcr |> Deserialize.Proto2.fromZcbLD InnerSample.Default
}

ys |> Seq.iter ignore
Expand Down
28 changes: 14 additions & 14 deletions Serialization.Test/TestClassSerialization.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@ module ClassSerialization =
member m.Serialize zcb = Serialize.toZeroCopyBuffer m zcb
member m.SerializeLengthDelimited () = Serialize.toArrayLD m

static member Deserialize buf = buf |> Deserialize.fromArray (InnerMessage())
static member Deserialize buf = buf |> Deserialize.fromArraySegment (InnerMessage())
static member Deserialize zcb = zcb |> Deserialize.fromZeroCopyBuffer (InnerMessage())
static member Deserialize raw = raw |> Deserialize.fromRawField (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.fromArrayLD (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.fromArraySegmentLD (InnerMessage())
static member DeserializeLengthDelimited zcb = zcb |> Deserialize.fromZcbLD (InnerMessage())
static member Deserialize buf = buf |> Deserialize.Proto2.fromArray (InnerMessage())
static member Deserialize buf = buf |> Deserialize.Proto2.fromArraySegment (InnerMessage())
static member Deserialize zcb = zcb |> Deserialize.Proto2.fromZeroCopyBuffer (InnerMessage())
static member Deserialize raw = raw |> Deserialize.Proto2.fromRawField (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.Proto2.fromArrayLD (InnerMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.Proto2.fromArraySegmentLD (InnerMessage())
static member DeserializeLengthDelimited zcb = zcb |> Deserialize.Proto2.fromZcbLD (InnerMessage())


[<Fact>]
Expand Down Expand Up @@ -155,13 +155,13 @@ module ClassSerialization =
member m.Serialize zcb = Serialize.toZeroCopyBuffer m zcb
member m.SerializeLengthDelimited () = Serialize.toArrayLD m

static member Deserialize buf = buf |> Deserialize.fromArray (OuterMessage())
static member Deserialize buf = buf |> Deserialize.fromArraySegment (OuterMessage())
static member Deserialize zcb = zcb |> Deserialize.fromZeroCopyBuffer (OuterMessage())
static member Deserialize raw = raw |> Deserialize.fromRawField (OuterMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.fromArrayLD (OuterMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.fromArraySegmentLD (OuterMessage())
static member DeserializeLengthDelimited zcb = zcb |> Deserialize.fromZcbLD (OuterMessage())
static member Deserialize buf = buf |> Deserialize.Proto2.fromArray (OuterMessage())
static member Deserialize buf = buf |> Deserialize.Proto2.fromArraySegment (OuterMessage())
static member Deserialize zcb = zcb |> Deserialize.Proto2.fromZeroCopyBuffer (OuterMessage())
static member Deserialize raw = raw |> Deserialize.Proto2.fromRawField (OuterMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.Proto2.fromArrayLD (OuterMessage())
static member DeserializeLengthDelimited buf = buf |> Deserialize.Proto2.fromArraySegmentLD (OuterMessage())
static member DeserializeLengthDelimited zcb = zcb |> Deserialize.Proto2.fromZcbLD (OuterMessage())


[<Fact>]
Expand Down
133 changes: 129 additions & 4 deletions Serialization.Test/TestSerialization.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ module RecordSerialization =
0x54uy; 0x65uy; 0x73uy; 0x74uy; 0x20uy; 0x6duy; 0x65uy; 0x73uy; 0x73uy; 0x61uy; 0x67uy; 0x65uy
// value "Test message"
|]
let msg = buf |> Deserialize.fromArray InnerMessage.Default
let msg = buf |> Deserialize.Proto2.fromArray InnerMessage.Default
msg.id |> should equal 99
msg.name |> should equal "Test message"

Expand All @@ -79,7 +79,7 @@ module RecordSerialization =
0x54uy; 0x65uy; 0x73uy; 0x74uy; 0x20uy; 0x6duy; 0x65uy; 0x73uy; 0x73uy; 0x61uy; 0x67uy; 0x65uy
// value "Test message"
|]
fun () -> buf |> Deserialize.fromArray InnerMessage.Default |> ignore
fun () -> buf |> Deserialize.Proto2.fromArray InnerMessage.Default |> ignore
|> should throw typeof<Froto.Serialization.SerializerException>

[<Fact>]
Expand Down Expand Up @@ -119,7 +119,7 @@ module RecordSerialization =

static member DecoderRing =
[ 1 , fun m rawField -> { m with id = Decode.toInt32 rawField } : OuterMessage
42, fun m rawField -> { m with inner = Deserialize.optionalMessage InnerMessage.Default rawField} : OuterMessage
42, fun m rawField -> { m with inner = Deserialize.Proto2.optionalMessage InnerMessage.Default rawField} : OuterMessage
43, fun m rawField -> { m with hasMore = Decode.toBool rawField } : OuterMessage
]
|> Map.ofList
Expand All @@ -146,7 +146,7 @@ module RecordSerialization =
0xD8uy ||| 0uy; 0x02uy; // tag: fldnum=43, varint
0x01uy; // value true
|]
let msg = buf |> Deserialize.fromArray OuterMessage.Default
let msg = buf |> Deserialize.Proto2.fromArray OuterMessage.Default
msg.id |> should equal 21
msg.inner.IsSome |> should equal true
msg.inner.Value.id |> should equal 99
Expand Down Expand Up @@ -175,4 +175,129 @@ module RecordSerialization =
0xD8uy ||| 0uy; 0x02uy; // tag: fldnum=43, varint
0x01uy; // value true
|]

///If you require the collection of unknown fields in your record type definitions then you can
///use the following definition as a guide.
///Note the _unknownFields definition as part of the record definition, zero entry in the DecoderRing,
///the UnknownFields method and DecodeFixup has to reverse the order of the _unknownFields list.
type Proto3Message = {
id : int32
name : string
_unknownFields : RawField list
}
with
static member Default = {
id = 0
name = ""
_unknownFields = List.empty
}

static member Serializer (m, zcb) =
(m.id |> Encode.fromVarint 1) >>
(m.name |> Encode.fromString 2) >>
(m._unknownFields|> Encode.fromRawFields)
<| zcb

static member DecoderRing =
[
0, fun m rawField -> { m with _unknownFields = rawField :: m._unknownFields } : Proto3Message
1, fun m rawField -> { m with id = rawField |> Decode.toInt32 } : _
2, fun m rawField -> { m with name = rawField |> Decode.toString } : _
]
|> Map.ofList

static member DecodeFixup m =
{ m with _unknownFields = List.rev m._unknownFields }

static member UnknownFields m =
m._unknownFields

[<Fact>]
let ``Serialize proto3 message`` () =
let msg = { Proto3Message.Default with
id = 5
name = "TEST" }
let serialised = msg |> Serialize.toArray
printfn "%A" serialised
serialised
|> should equal
[|
1uy <<< 3 ||| 0uy; 5uy // field 1, type 0; value 5
2uy <<< 3 ||| 2uy; // field 2, type 2
4uy; // length
84uy; 69uy; 83uy; 84uy // TEST
|]

[<Fact>]
let ``Deserialize proto3 message`` () =
let buf =
[|
1uy <<< 3 ||| 0uy; 5uy // field 1, type 0; value 5
2uy <<< 3 ||| 2uy; // field 2, type 2
4uy; // length
84uy; 69uy; 83uy; 84uy // TEST
|]
let msg = buf |> Deserialize.Proto3.fromArray Proto3Message.Default
msg.id |> should equal 5
msg.name |> should equal "TEST"

[<Fact>]
let ``Deserialize proto3 message with extra field`` () =
let buf =
[|
1uy <<< 3 ||| 0uy; 5uy // field 1, type 0; value 5
2uy <<< 3 ||| 2uy; // field 2, type 2
4uy; // length
84uy; 69uy; 83uy; 84uy // TEST
3uy <<< 3 ||| 0uy; 42uy //extra field number 3, type varint, value 42
|]
let msg = buf |> Deserialize.Proto3.fromArray Proto3Message.Default
msg.id |> should equal 5
msg.name |> should equal "TEST"
let unknown =
msg._unknownFields
|> List.tryHead
|> Option.toList
|> List.map (fun v -> v.FieldNum, v.WireType)
unknown |> should equal [3, WireType.Varint]

///If you dont require the optional collection of Unknown Fields then your record type definitions
///can be described like the following.
///Note the absence of a zero entry in the DecoderRing, no UnknownFields and DecodeFixup has no work to do.
type Proto3MessageNoUnknown = {
id : int32
name : string
}
with
static member Default = {
id = 0
name = ""
}

static member Serializer (m, zcb) =
(m.id |> Encode.fromVarint 1) >>
(m.name |> Encode.fromString 2)
<| zcb

static member DecoderRing =
[
1, fun m rawField -> { m with id = rawField |> Decode.toInt32 } : Proto3MessageNoUnknown
2, fun m rawField -> { m with name = rawField |> Decode.toString } : _
]
|> Map.ofList

static member DecodeFixup m = m

[<Fact>]
let ``Deserialize proto3 message discarding extra field`` () =
let buf =
[|
1uy <<< 3 ||| 0uy; 5uy // field 1, type 0; value 5
2uy <<< 3 ||| 2uy; // field 2, type 2
4uy; // length
84uy; 69uy; 83uy; 84uy // TEST
3uy <<< 3 ||| 0uy; 42uy //extra field number 3, type varint, value 42
|]
let msg = buf |> Deserialize.Proto3.fromArray Proto3MessageNoUnknown.Default
msg.id |> should equal 5
msg.name |> should equal "TEST"
Loading

0 comments on commit ca65a0f

Please sign in to comment.