diff --git a/src/MsBuildPipeLogger.Server/NamedPipeLoggerServer.cs b/src/MsBuildPipeLogger.Server/NamedPipeLoggerServer.cs
index 2fa5181..87096b5 100644
--- a/src/MsBuildPipeLogger.Server/NamedPipeLoggerServer.cs
+++ b/src/MsBuildPipeLogger.Server/NamedPipeLoggerServer.cs
@@ -1,6 +1,7 @@
 using System;
 using System.IO.Pipes;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace MsBuildPipeLogger
 {
@@ -9,6 +10,8 @@ namespace MsBuildPipeLogger
     /// </summary>
     public class NamedPipeLoggerServer : PipeLoggerServer<NamedPipeServerStream>
     {
+        public string PipeName { get; }
+
         /// <summary>
         /// Creates a named pipe server for receiving MSBuild logging events.
         /// </summary>
@@ -24,11 +27,69 @@ public NamedPipeLoggerServer(string pipeName)
         /// <param name="pipeName">The name of the pipe to create.</param>
         /// <param name="cancellationToken">A <see cref="CancellationToken"/> that will cancel read operations if triggered.</param>
         public NamedPipeLoggerServer(string pipeName, CancellationToken cancellationToken)
-            : base(new NamedPipeServerStream(pipeName, PipeDirection.In, -1, PipeTransmissionMode.Byte, PipeOptions.Asynchronous), cancellationToken)
+            : base(new NamedPipeServerStream(pipeName, PipeDirection.In), cancellationToken)
+        {
+            PipeName = pipeName;
+            CancellationToken.Register(CancelConnectionWait);
+        }
+
+        protected override void Connect()
         {
+            PipeStream.WaitForConnection();
+            _connected.Set();
         }
 
-        protected override void Connect() =>
-            CancellationToken.Try(() => PipeStream.WaitForConnectionAsync(CancellationToken).Wait());
+        private readonly InterlockedBool _connected = new InterlockedBool(false);
+
+        private void CancelConnectionWait()
+        {
+            if(!_connected.Set())
+            {
+                // This is a crazy hack that stops the WaitForConnection by connecting a dummy client
+                // We have to do it this way instead of checking for .IsConnected because if we connect
+                // and then disconnect very quickly, .IsConnected will never observe as true and we'll lock
+                using (NamedPipeClientStream pipeStream = new NamedPipeClientStream(".", PipeName, PipeDirection.Out))
+                {
+                    pipeStream.Connect();
+                }
+            }
+        }
+    }
+
+    internal class InterlockedBool
+    {
+        private volatile int _set;
+
+        public InterlockedBool()
+        {
+            _set = 0;
+        }
+
+        public InterlockedBool(bool initialState)
+        {
+            _set = initialState ? 1 : 0;
+        }
+
+        // Returns the previous switch state of the switch
+        public bool Set()
+        {
+#pragma warning disable 420
+            return Interlocked.Exchange(ref _set, 1) != 0;
+#pragma warning restore 420
+        }
+
+        // Returns the previous switch state of the switch
+        public bool Unset()
+        {
+#pragma warning disable 420
+            return Interlocked.Exchange(ref _set, 0) != 0;
+#pragma warning restore 420
+        }
+
+        // Returns the current state
+        public static implicit operator bool(InterlockedBool interlockedBool)
+        {
+            return interlockedBool._set != 0;
+        }
     }
 }