separate based on TLS and no-TLS

benchmark/BDN.benchmark/Embedded/EmbeddedNetworkHandler.cs

@@ -12,8 +12,11 @@ namespace Embedded.server
 {
     internal class EmbeddedNetworkHandler : NetworkHandler<GarnetServerEmbedded, EmbeddedNetworkSender>
     {
+        readonly bool useTLS;
+
         public EmbeddedNetworkHandler(GarnetServerEmbedded serverHook, EmbeddedNetworkSender networkSender, NetworkBufferSettings networkBufferSettings, LimitedFixedBufferPool networkPool, bool useTLS, IMessageConsumer messageConsumer = null, ILogger logger = null) : base(serverHook, networkSender, networkBufferSettings, networkPool, useTLS, messageConsumer, logger)
         {
+            this.useTLS = useTLS;
         }
 
         public override string RemoteEndpointName => throw new NotImplementedException();
@@ -25,12 +28,15 @@ public override void Dispose()
 
         public override bool TryClose() => throw new NotImplementedException();
 
-        public void Send(Request request)
+        public async ValueTask Send(Request request)
         {
             networkReceiveBuffer = request.buffer;
             unsafe { networkReceiveBufferPtr = request.bufferPtr; }
 
-            OnNetworkReceive(request.buffer.Length);
+            if (useTLS)
+                await OnNetworkReceiveWithTLSAsync(request.buffer.Length);
+            else
+                OnNetworkReceiveWithoutTLS(request.buffer.Length);
 
             Debug.Assert(networkBytesRead == 0);
             Debug.Assert(networkReadHead == 0);

benchmark/BDN.benchmark/Network/BasicOperations.cs

@@ -22,9 +22,9 @@ public override void GlobalSetup()
         }
 
         [Benchmark]
-        public void InlinePing()
+        public async ValueTask InlinePing()
         {
-            Send(ping);
+            await Send(ping);
         }
     }
 }

benchmark/BDN.benchmark/Network/NetworkBase.cs

@@ -64,7 +64,7 @@ public virtual void GlobalCleanup()
             server.Dispose();
         }
 
-        protected void Send(Request request) => networkHandler.Send(request);
+        protected ValueTask Send(Request request) => networkHandler.Send(request);
 
         protected unsafe void SetupOperation(ref Request request, ReadOnlySpan<byte> operation, int batchSize = batchSize)
         {

libs/common/Networking/NetworkHandler.cs

@@ -267,38 +267,25 @@ async Task AuthenticateAsClientAsync(SslClientAuthenticationOptions sslClientOpt
             }
         }
 
-        public void OnNetworkReceive(int bytesTransferred)
+        public unsafe void OnNetworkReceiveWithoutTLS(int bytesTransferred)
         {
             networkBytesRead += bytesTransferred;
             transportReceiveBuffer = networkReceiveBuffer;
-            unsafe
-            {
-                transportReceiveBufferPtr = networkReceiveBufferPtr;
-            }
+            transportReceiveBufferPtr = networkReceiveBufferPtr;
             transportBytesRead = networkBytesRead;
 
-            // We do not have an active read task, so we will process on the network thread
+            // Process non-TLS code on the synchronous thread
             Process();
+
             EndTransformNetworkToTransport();
             UpdateNetworkBuffers();
         }
 
-        private void UpdateNetworkBuffers()
-        {
-            // Shift network buffer after processing is done
-            if (networkReadHead > 0)
-                ShiftNetworkReceiveBuffer();
-
-            // Double network buffer if out of space after processing is complete
-            if (networkBytesRead == networkReceiveBuffer.Length)
-                DoubleNetworkReceiveBuffer();
-        }
-
         /// <summary>
         /// On network receive
         /// </summary>
         /// <param name="bytesTransferred">Number of bytes transferred</param>
-        public async ValueTask OnNetworkReceiveWithTLS(int bytesTransferred)
+        public async ValueTask OnNetworkReceiveWithTLSAsync(int bytesTransferred)
         {
             // Wait for SslStream async processing to complete, if any (e.g., authentication phase)
             while (readerStatus == TlsReaderStatus.Active)
@@ -334,6 +321,17 @@ public async ValueTask OnNetworkReceiveWithTLS(int bytesTransferred)
             UpdateNetworkBuffers();
         }
 
+        void UpdateNetworkBuffers()
+        {
+            // Shift network buffer after processing is done
+            if (networkReadHead > 0)
+                ShiftNetworkReceiveBuffer();
+
+            // Double network buffer if out of space after processing is complete
+            if (networkBytesRead == networkReceiveBuffer.Length)
+                DoubleNetworkReceiveBuffer();
+        }
+
         [MethodImpl(MethodImplOptions.NoInlining)]
         static void ThrowInvalidOperationException(string message)
             => throw new InvalidOperationException(message);

libs/common/Networking/TcpNetworkHandlerBase.cs

@@ -53,28 +53,28 @@ public TcpNetworkHandlerBase(TServerHook serverHook, TNetworkSender networkSende
         /// <inheritdoc />
         public override void Start(SslServerAuthenticationOptions tlsOptions = null, string remoteEndpointName = null, CancellationToken token = default)
         {
-            Start();
+            Start(tlsOptions != null);
             base.Start(tlsOptions, remoteEndpointName, token);
         }
 
         /// <inheritdoc />
         public override async Task StartAsync(SslServerAuthenticationOptions tlsOptions = null, string remoteEndpointName = null, CancellationToken token = default)
         {
-            Start();
+            Start(tlsOptions != null);
             await base.StartAsync(tlsOptions, remoteEndpointName, token).ConfigureAwait(false);
         }
 
         /// <inheritdoc />
         public override void Start(SslClientAuthenticationOptions tlsOptions, string remoteEndpointName = null, CancellationToken token = default)
         {
-            Start();
+            Start(tlsOptions != null);
             base.Start(tlsOptions, remoteEndpointName, token);
         }
 
         /// <inheritdoc />
         public override async Task StartAsync(SslClientAuthenticationOptions tlsOptions, string remoteEndpointName = null, CancellationToken token = default)
         {
-            Start();
+            Start(tlsOptions != null);
             await base.StartAsync(tlsOptions, remoteEndpointName, token).ConfigureAwait(false);
         }
 
@@ -103,17 +103,22 @@ public override bool TryClose()
             return true;
         }
 
-        void Start()
+        void Start(bool useTLS)
         {
             var receiveEventArgs = new SocketAsyncEventArgs { AcceptSocket = socket };
             receiveEventArgs.SetBuffer(networkReceiveBuffer, 0, networkReceiveBuffer.Length);
-            receiveEventArgs.Completed += RecvEventArg_Completed;
+            receiveEventArgs.Completed += useTLS ? RecvEventArgCompletedWithTLS : RecvEventArgCompletedWithoutTLS;
 
             // If the client already have packets, avoid handling it here on the handler so we don't block future accepts.
             try
             {
                 if (!socket.ReceiveAsync(receiveEventArgs))
-                    Task.Run(() => RecvEventArg_Completed(null, receiveEventArgs));
+                {
+                    if (useTLS)
+                        Task.Run(() => RecvEventArgCompletedWithTLS(null, receiveEventArgs));
+                    else
+                        Task.Run(() => RecvEventArgCompletedWithoutTLS(null, receiveEventArgs));
+                }
             }
             catch (Exception ex)
             {
@@ -135,20 +140,13 @@ void Dispose(SocketAsyncEventArgs e)
             e.Dispose();
         }
 
-        void RecvEventArg_Completed(object sender, SocketAsyncEventArgs e)
-        {
-            // Complete receive event and release thread while we process data async
-            if (this.useTLS)
-            {
-                _ = HandleReceiveAsync(sender, e);
-            }
-            else
-            {
-                HandleReceiveSync(sender, e);
-            }
-        }
+        void RecvEventArgCompletedWithTLS(object sender, SocketAsyncEventArgs e) =>
+            _ = HandleReceiveWithTLSAsync(sender, e);
+
+        void RecvEventArgCompletedWithoutTLS(object sender, SocketAsyncEventArgs e) =>
+            HandleReceiveWithoutTLS(sender, e);
 
-        private void HandleReceiveSync(object sender, SocketAsyncEventArgs e)
+        private void HandleReceiveWithoutTLS(object sender, SocketAsyncEventArgs e)
         {
             try
             {
@@ -160,7 +158,7 @@ private void HandleReceiveSync(object sender, SocketAsyncEventArgs e)
                         Dispose(e);
                         break;
                     }
-                    OnNetworkReceive(e.BytesTransferred);
+                    OnNetworkReceiveWithoutTLS(e.BytesTransferred);
                     e.SetBuffer(networkReceiveBuffer, networkBytesRead, networkReceiveBuffer.Length - networkBytesRead);
                 } while (!e.AcceptSocket.ReceiveAsync(e));
             }
@@ -170,7 +168,7 @@ private void HandleReceiveSync(object sender, SocketAsyncEventArgs e)
             }
         }
 
-        private async ValueTask HandleReceiveAsync(object sender, SocketAsyncEventArgs e)
+        private async ValueTask HandleReceiveWithTLSAsync(object sender, SocketAsyncEventArgs e)
         {
             try
             {
@@ -182,7 +180,7 @@ private async ValueTask HandleReceiveAsync(object sender, SocketAsyncEventArgs e
                         Dispose(e);
                         break;
                     }
-                    var receiveTask = OnNetworkReceiveWithTLS(e.BytesTransferred);
+                    var receiveTask = OnNetworkReceiveWithTLSAsync(e.BytesTransferred);
                     if (!receiveTask.IsCompletedSuccessfully)
                     {
                         await receiveTask;