From 11a32886c9ca47e39637068caa28374e7b01ba98 Mon Sep 17 00:00:00 2001 From: PixeL Date: Tue, 3 Sep 2019 02:30:58 -0500 Subject: [PATCH 1/5] Implement ZLIB-Stream --- source/droid/gateway/gateway.d | 36 ++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/source/droid/gateway/gateway.d b/source/droid/gateway/gateway.d index 2c3c8f6..4a7e501 100644 --- a/source/droid/gateway/gateway.d +++ b/source/droid/gateway/gateway.d @@ -6,7 +6,9 @@ import core.time, std.stdio, std.typecons, std.random, - std.experimental.logger; + std.experimental.logger, + std.array, + std.zlib; import vibe.core.core, vibe.http.common, @@ -19,7 +21,8 @@ import droid.exception, droid.api, droid.gateway.opcode, droid.gateway.packet, - droid.data.event_type; + droid.data.event_type, + droid.gateway.compression; final class Gateway { @@ -32,7 +35,9 @@ final class Gateway private immutable OpcodeHandlerMap OPCODE_MAPPING; - private immutable string gatewayUrl_; + private string gatewayUrl_; + private immutable CompressionType compressionType = CompressionType.ZLIB_STREAM; + private Decompressor decompressor = null; private API api_; private WebSocket ws_; @@ -61,6 +66,18 @@ final class Gateway void connect(in bool blocking = true, in bool reconnecting = false) { + if (compressionType != CompressionType.NONE) { + gatewayUrl_ = gatewayUrl_ ~ "&compress=" ~ compressionType; + switch (compressionType) { + case CompressionType.ZLIB_STREAM: + decompressor = new ZLibStream(); + break; + default: + throw new DroidException("Compression type not supported!"); + + } + } + if (!tryConnect(gatewayUrl_)) { logger_.tracef("Could not connect to given gateway url %s, using API", gatewayUrl_); tryConnect(api_.getGatewayUrl(), true); @@ -131,12 +148,23 @@ final class Gateway return true; } + public void kys() { + ws_.close(); + } + private void handleEvents() { assert(ws_ && ws_.connected); while (ws_.waitForData()) { - const packet = parseMessage(ws_.receiveText()); + auto data = ""; + + if (decompressor) { + data = decompressor.read(ws_.receiveBinary()); + } else + data = ws_.receiveText(); + + const packet = parseMessage(data); auto opcodeHandler = packet.opcode in OPCODE_MAPPING; if (opcodeHandler) { From c05ed387b6313114c6a9d3345bed88c5033a21dd Mon Sep 17 00:00:00 2001 From: PixeL Date: Tue, 3 Sep 2019 02:32:47 -0500 Subject: [PATCH 2/5] Woops :) --- source/droid/gateway/gateway.d | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/source/droid/gateway/gateway.d b/source/droid/gateway/gateway.d index 4a7e501..5170da4 100644 --- a/source/droid/gateway/gateway.d +++ b/source/droid/gateway/gateway.d @@ -148,10 +148,6 @@ final class Gateway return true; } - public void kys() { - ws_.close(); - } - private void handleEvents() { assert(ws_ && ws_.connected); @@ -159,9 +155,9 @@ final class Gateway while (ws_.waitForData()) { auto data = ""; - if (decompressor) { + if (decompressor) data = decompressor.read(ws_.receiveBinary()); - } else + else data = ws_.receiveText(); const packet = parseMessage(data); From f36997f3996ef593cf2415fec1bad3b4c581c57a Mon Sep 17 00:00:00 2001 From: PixeL Date: Tue, 3 Sep 2019 02:33:59 -0500 Subject: [PATCH 3/5] Push missing compression file --- source/droid/gateway/compression.d | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 source/droid/gateway/compression.d diff --git a/source/droid/gateway/compression.d b/source/droid/gateway/compression.d new file mode 100644 index 0000000..caf3761 --- /dev/null +++ b/source/droid/gateway/compression.d @@ -0,0 +1,37 @@ +module droid.gateway.compression; + +import droid.exception; +import std.zlib, + std.stdio, + std.conv : to; + +enum CompressionType : string { + NONE = "", + ZLIB = "zlib", + ZLIB_STREAM = "zlib-stream" +} +class Decompressor { + string read(ubyte[] data) { + throw new DroidException("Compression type not supported!"); + } +} + +class ZLibStream : Decompressor { + const ulong[] ZLIB_SUFFIX = [0x0, 0x0, 0xFF, 0xFF]; + UnCompress decompressor; + + this() { + decompressor = new UnCompress(HeaderFormat.deflate); + } + + override string read(ubyte[] data) { + if (data[$-4..$] != ZLIB_SUFFIX) { + throw new DroidException("ZLib-Stream compression enabled but invalid data was recieved!"); + } + + string decompressed = to!string(decompressor.uncompress(data)); + decompressor.flush(); + + return decompressed; + } +} From 85e1cadbe82832d65be56a02a68964363a54ef43 Mon Sep 17 00:00:00 2001 From: PixeL Date: Thu, 13 Aug 2020 10:25:24 -0500 Subject: [PATCH 4/5] Fix zlib-stream --- source/droid/gateway/compression.d | 17 ++++++++++++++--- source/droid/gateway/gateway.d | 3 +++ source/droid/gateway/package.d | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/source/droid/gateway/compression.d b/source/droid/gateway/compression.d index caf3761..e22407c 100644 --- a/source/droid/gateway/compression.d +++ b/source/droid/gateway/compression.d @@ -17,20 +17,31 @@ class Decompressor { } class ZLibStream : Decompressor { - const ulong[] ZLIB_SUFFIX = [0x0, 0x0, 0xFF, 0xFF]; + const ubyte[] ZLIB_SUFFIX = [0x0, 0x0, 0xFF, 0xFF]; UnCompress decompressor; + ubyte[] buffer; + this() { decompressor = new UnCompress(HeaderFormat.deflate); } + /* + * Reads a zlib stream from the websocket + * This will append the data to a buffer, + * returning nothing if the data is not a full zlib frame + * otherwise, returning the decompressed string. + */ override string read(ubyte[] data) { + buffer ~= data; + if (data[$-4..$] != ZLIB_SUFFIX) { - throw new DroidException("ZLib-Stream compression enabled but invalid data was recieved!"); + return ""; } - string decompressed = to!string(decompressor.uncompress(data)); + string decompressed = to!string(decompressor.uncompress(buffer)); decompressor.flush(); + buffer = null; return decompressed; } diff --git a/source/droid/gateway/gateway.d b/source/droid/gateway/gateway.d index 5170da4..4d4f08c 100644 --- a/source/droid/gateway/gateway.d +++ b/source/droid/gateway/gateway.d @@ -160,6 +160,9 @@ final class Gateway else data = ws_.receiveText(); + // The data isn't complete (not a full zlib message, or something borked) + if (data == "") return; + const packet = parseMessage(data); auto opcodeHandler = packet.opcode in OPCODE_MAPPING; diff --git a/source/droid/gateway/package.d b/source/droid/gateway/package.d index 42b2360..9769d38 100644 --- a/source/droid/gateway/package.d +++ b/source/droid/gateway/package.d @@ -4,4 +4,5 @@ public { import droid.gateway.opcode; import droid.gateway.packet; import droid.gateway.gateway; + import droid.gateway.compression; } From f49eaf8ba84f295e0ded3f2ff08da9e145b57bf2 Mon Sep 17 00:00:00 2001 From: PixeL Date: Thu, 13 Aug 2020 10:56:18 -0500 Subject: [PATCH 5/5] Large payload zlib --- source/droid/gateway/compression.d | 7 +++++++ source/droid/gateway/gateway.d | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/source/droid/gateway/compression.d b/source/droid/gateway/compression.d index e22407c..e73ea5e 100644 --- a/source/droid/gateway/compression.d +++ b/source/droid/gateway/compression.d @@ -10,12 +10,19 @@ enum CompressionType : string { ZLIB = "zlib", ZLIB_STREAM = "zlib-stream" } + class Decompressor { string read(ubyte[] data) { throw new DroidException("Compression type not supported!"); } } +class ZLib : Decompressor { + override string read(ubyte[] data) { + return to!string(new UnCompress(HeaderFormat.deflate).uncompress(data)); + } +} + class ZLibStream : Decompressor { const ubyte[] ZLIB_SUFFIX = [0x0, 0x0, 0xFF, 0xFF]; UnCompress decompressor; diff --git a/source/droid/gateway/gateway.d b/source/droid/gateway/gateway.d index 4d4f08c..c0b511f 100644 --- a/source/droid/gateway/gateway.d +++ b/source/droid/gateway/gateway.d @@ -67,11 +67,14 @@ final class Gateway void connect(in bool blocking = true, in bool reconnecting = false) { if (compressionType != CompressionType.NONE) { - gatewayUrl_ = gatewayUrl_ ~ "&compress=" ~ compressionType; switch (compressionType) { case CompressionType.ZLIB_STREAM: + gatewayUrl_ = gatewayUrl_ ~ "&compress=" ~ compressionType; decompressor = new ZLibStream(); break; + case CompressionType.ZLIB: + decompressor = new ZLib(); + break; default: throw new DroidException("Compression type not supported!"); @@ -118,6 +121,7 @@ final class Gateway opcodeIdentifyHandle(Json([ "token": Json(api_.token), + "compress": Json(compressionType == CompressionType.ZLIB), "properties": Json([ "$os": Json(osName), "$browser": Json("droid"), @@ -155,9 +159,15 @@ final class Gateway while (ws_.waitForData()) { auto data = ""; - if (decompressor) - data = decompressor.read(ws_.receiveBinary()); - else + if (decompressor) { + // TODO: This may be meh. + try { + data = decompressor.read(ws_.receiveBinary()); + } catch (WebSocketException e) { + // The data isn't compressed or something went wrong, Yeet it to text! + data = ws_.receiveText(); + } + } else data = ws_.receiveText(); // The data isn't complete (not a full zlib message, or something borked)