UTF-8 WIP

.github/workflows/build.yml

@@ -15,9 +15,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Setup .NET
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
-          dotnet-version: 7.0.x
+          dotnet-version: 8.0.x
 
       - name: Restore
         run: dotnet restore src/ZeroLog.sln

src/Directory.Build.props

@@ -3,7 +3,7 @@
     <WarningLevel>4</WarningLevel>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-    <LangVersion>11.0</LangVersion>
+    <LangVersion>12.0</LangVersion>
     <Prefer32Bit>false</Prefer32Bit>
     <DefaultItemExcludes>$(DefaultItemExcludes);*.DotSettings;*.ncrunchproject</DefaultItemExcludes>
     <DebugType>embedded</DebugType>

src/ZeroLog.Analyzers.Tests/ZeroLog.Analyzers.Tests.csproj

@@ -15,9 +15,9 @@
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.CodeFix.Testing" Version="1.1.1" />
     <PackageReference Include="Microsoft.CodeAnalysis.Testing.Verifiers.NUnit" Version="1.1.1" />
 
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
-    <PackageReference Include="NUnit" Version="3.13.3" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.3.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="NUnit" Version="3.14.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
   </ItemGroup>
 
 </Project>

src/ZeroLog.Impl.Base/Log.cs

@@ -13,12 +13,18 @@ public sealed partial class Log
     private LogLevel _logLevel = LogLevel.None;
 
     internal string Name { get; }
+    internal byte[] NameUtf8 { get; }
+
     internal string CompactName { get; }
+    internal byte[] CompactNameUtf8 { get; }
 
     internal Log(string name)
     {
         Name = name;
+        NameUtf8 = Encoding.UTF8.GetBytes(Name);
+
         CompactName = GetCompactName(name);
+        CompactNameUtf8 = Encoding.UTF8.GetBytes(CompactName);
     }
 
     /// <summary>

src/ZeroLog.Impl.Full/Appenders/StreamAppender.cs

@@ -13,9 +13,10 @@ public abstract class StreamAppender : Appender
 {
     private byte[] _byteBuffer = Array.Empty<byte>();
 
+    private readonly Formatter _formatter = new DefaultFormatter();
     private Encoding _encoding = Encoding.UTF8;
+    private Utf8Formatter? _utf8Formatter;
     private bool _useSpanGetBytes;
-    private Formatter? _formatter;
 
     /// <summary>
     /// The stream to write to.
@@ -30,6 +31,9 @@ protected internal Encoding Encoding
         get => _encoding;
         set
         {
+            if (ReferenceEquals(value, _encoding))
+                return;
+
             _encoding = value;
             UpdateEncodingSpecificData();
         }
@@ -40,8 +44,15 @@ protected internal Encoding Encoding
     /// </summary>
     public Formatter Formatter
     {
-        get => _formatter ??= new DefaultFormatter();
-        init => _formatter = value;
+        get => _formatter;
+        init
+        {
+            if (ReferenceEquals(value, _formatter))
+                return;
+
+            _formatter = value;
+            UpdateEncodingSpecificData();
+        }
     }
 
     /// <summary>
@@ -64,21 +75,25 @@ public override void Dispose()
     /// <inheritdoc/>
     public override void WriteMessage(LoggedMessage message)
     {
-        if (Stream is null)
+        if (Stream is not { } stream)
             return;
 
-        if (_useSpanGetBytes)
+        if (_utf8Formatter is { } utf8Formatter)
+        {
+            stream.Write(utf8Formatter.FormatMessage(message));
+        }
+        else if (_useSpanGetBytes)
         {
-            var chars = Formatter.FormatMessage(message);
+            var chars = _formatter.FormatMessage(message);
             var byteCount = _encoding.GetBytes(chars, _byteBuffer);
-            Stream.Write(_byteBuffer, 0, byteCount);
+            stream.Write(_byteBuffer, 0, byteCount);
         }
         else
         {
-            Formatter.FormatMessage(message);
-            var charBuffer = Formatter.GetBuffer(out var charCount);
+            _formatter.FormatMessage(message);
+            var charBuffer = _formatter.GetBuffer(out var charCount);
             var byteCount = _encoding.GetBytes(charBuffer, 0, charCount, _byteBuffer, 0);
-            Stream.Write(_byteBuffer, 0, byteCount);
+            stream.Write(_byteBuffer, 0, byteCount);
         }
     }
 
@@ -91,14 +106,23 @@ public override void Flush()
 
     private void UpdateEncodingSpecificData()
     {
-        var maxBytes = _encoding.GetMaxByteCount(LogManager.OutputBufferSize);
+        if (_encoding is UTF8Encoding && _formatter.AsUtf8Formatter() is { } utf8Formatter)
+        {
+            // Fast path
+            _utf8Formatter = utf8Formatter;
+            return;
+        }
+
+        _utf8Formatter = null;
 
         // The base Encoding class allocates buffers in all non-abstract GetBytes overloads in order to call the abstract
         // GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) in the end.
         // If an encoding overrides the Span version of GetBytes, we assume it avoids this allocation
         // and it skips safety checks as those are guaranteed by the Span struct. In that case, we can call this overload directly.
         _useSpanGetBytes = OverridesSpanGetBytes(_encoding.GetType());
 
+        var maxBytes = _encoding.GetMaxByteCount(LogManager.OutputBufferSize);
+
         if (_byteBuffer.Length < maxBytes)
             _byteBuffer = GC.AllocateUninitializedArray<byte>(maxBytes);
     }

src/ZeroLog.Impl.Full/Configuration/ZeroLogConfiguration.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
+using System.Text;
 using ZeroLog.Appenders;
 using ZeroLog.Formatting;
 
@@ -16,6 +17,9 @@ public sealed class ZeroLogConfiguration
 
     private LoggerConfigurationCollection _loggers = new();
 
+    private string _nullDisplayString = "null";
+    private string _truncatedMessageSuffix = " [TRUNCATED]";
+
     internal event Action? ApplyChangesRequested;
 
     /// <summary>
@@ -86,15 +90,35 @@ public bool UseBackgroundThread
     /// <remarks>
     /// Default: "null"
     /// </remarks>
-    public string NullDisplayString { get; set; } = "null";
+    public string NullDisplayString
+    {
+        get => _nullDisplayString;
+        set
+        {
+            _nullDisplayString = value;
+            NullDisplayStringUtf8 = Encoding.UTF8.GetBytes(value);
+        }
+    }
+
+    internal byte[] NullDisplayStringUtf8 { get; private set; }
 
     /// <summary>
     /// The string which is appended to a message when it is truncated.
     /// </summary>
     /// <remarks>
     /// Default: " [TRUNCATED]"
     /// </remarks>
-    public string TruncatedMessageSuffix { get; set; } = " [TRUNCATED]";
+    public string TruncatedMessageSuffix
+    {
+        get => _truncatedMessageSuffix;
+        set
+        {
+            _truncatedMessageSuffix = value;
+            TruncatedMessageSuffixUtf8 = Encoding.UTF8.GetBytes(value);
+        }
+    }
+
+    internal byte[] TruncatedMessageSuffixUtf8 { get; private set; }
 
     /// <summary>
     /// The time an appender will be put into quarantine (not used to log messages) after it throws an exception.
@@ -121,6 +145,15 @@ public bool UseBackgroundThread
     /// </remarks>
     public ILoggerConfigurationCollection Loggers => _loggers;
 
+    /// <summary>
+    /// Creates a new ZeroLog configuration.
+    /// </summary>
+    public ZeroLogConfiguration()
+    {
+        NullDisplayStringUtf8 = Encoding.UTF8.GetBytes(NullDisplayString);
+        TruncatedMessageSuffixUtf8 = Encoding.UTF8.GetBytes(TruncatedMessageSuffix);
+    }
+
     /// <summary>
     /// Applies the changes made to this object since the call to <see cref="LogManager.Initialize"/>
     /// or the last call to <see cref="ApplyChanges"/>.

src/ZeroLog.Impl.Full/EnumArg.cs

@@ -1,8 +1,10 @@
 using System;
+using System.Buffers;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Text.Unicode;
 using ZeroLog.Configuration;
 using ZeroLog.Support;
 
@@ -44,6 +46,27 @@ public bool TryFormat(Span<char> destination, out int charsWritten, ZeroLogConfi
         return TryAppendNumericValue(destination, out charsWritten);
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public bool TryFormat(Span<byte> destination, out int bytesWritten, ZeroLogConfiguration config)
+    {
+        var enumString = GetUtf8String(config);
+
+        if (enumString != null)
+        {
+            if (enumString.Length <= destination.Length)
+            {
+                enumString.CopyTo(destination);
+                bytesWritten = enumString.Length;
+                return true;
+            }
+
+            bytesWritten = 0;
+            return false;
+        }
+
+        return TryAppendNumericValue(destination, out bytesWritten);
+    }
+
     [MethodImpl(MethodImplOptions.NoInlining)]
     private bool TryAppendNumericValue(Span<char> destination, out int charsWritten)
     {
@@ -53,11 +76,35 @@ private bool TryAppendNumericValue(Span<char> destination, out int charsWritten)
         return unchecked((long)_value).TryFormat(destination, out charsWritten, default, CultureInfo.InvariantCulture);
     }
 
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private bool TryAppendNumericValue(Span<byte> destination, out int bytesWritten)
+    {
+#if NET8_0_OR_GREATER
+        if (_value <= long.MaxValue || !EnumCache.IsEnumSigned(_typeHandle))
+            return _value.TryFormat(destination, out bytesWritten, default, CultureInfo.InvariantCulture);
+
+        return unchecked((long)_value).TryFormat(destination, out bytesWritten, default, CultureInfo.InvariantCulture);
+#else
+        Span<char> buffer = stackalloc char[32];
+
+        if (TryAppendNumericValue(buffer, out var charsWritten))
+            return Utf8.FromUtf16(buffer.Slice(0, charsWritten), destination, out _, out bytesWritten) == OperationStatus.Done;
+
+        bytesWritten = 0;
+        return false;
+#endif
+    }
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public string? GetString(ZeroLogConfiguration config)
         => EnumCache.GetString(_typeHandle, _value, out var enumRegistered)
            ?? GetStringSlow(enumRegistered, config);
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public byte[]? GetUtf8String(ZeroLogConfiguration config)
+        => EnumCache.GetUtf8String(_typeHandle, _value, out var enumRegistered)
+           ?? GetUtf8StringSlow(enumRegistered, config);
+
     [MethodImpl(MethodImplOptions.NoInlining)]
     private string? GetStringSlow(bool enumRegistered, ZeroLogConfiguration config)
     {
@@ -71,6 +118,19 @@ private bool TryAppendNumericValue(Span<char> destination, out int charsWritten)
         return EnumCache.GetString(_typeHandle, _value, out _);
     }
 
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private byte[]? GetUtf8StringSlow(bool enumRegistered, ZeroLogConfiguration config)
+    {
+        if (enumRegistered || !config.AutoRegisterEnums)
+            return null;
+
+        if (Type is not { } type)
+            return null;
+
+        LogManager.RegisterEnum(type);
+        return EnumCache.GetUtf8String(_typeHandle, _value, out _);
+    }
+
     public bool TryGetValue<T>(out T result)
         where T : unmanaged
     {