Additional support for string literals (#202)

src/Parlot/Compilation/ExpressionHelper.cs

@@ -31,6 +31,8 @@ public static class ExpressionHelper
     internal static readonly MethodInfo Scanner_ReadSingleQuotedString = typeof(Scanner).GetMethod(nameof(Parlot.Scanner.ReadSingleQuotedString), [])!;
     internal static readonly MethodInfo Scanner_ReadDoubleQuotedString = typeof(Scanner).GetMethod(nameof(Parlot.Scanner.ReadDoubleQuotedString), [])!;
     internal static readonly MethodInfo Scanner_ReadQuotedString = typeof(Scanner).GetMethod(nameof(Parlot.Scanner.ReadQuotedString), [])!;
+    internal static readonly MethodInfo Scanner_ReadBacktickString = typeof(Scanner).GetMethod(nameof(Parlot.Scanner.ReadBacktickString), [])!;
+    internal static readonly MethodInfo Scanner_ReadCustomString = typeof(Scanner).GetMethod(nameof(Parlot.Scanner.ReadQuotedString), [typeof(char[])])!;
 
     internal static readonly MethodInfo Cursor_Advance = typeof(Cursor).GetMethod(nameof(Parlot.Cursor.Advance), [])!;
     internal static readonly MethodInfo Cursor_AdvanceNoNewLines = typeof(Cursor).GetMethod(nameof(Parlot.Cursor.AdvanceNoNewLines), [typeof(int)])!;
@@ -64,6 +66,8 @@ public static class ExpressionHelper
 
     public static MethodCallExpression ReadSingleQuotedString(this CompilationContext context) => Expression.Call(context.Scanner(), Scanner_ReadSingleQuotedString);
     public static MethodCallExpression ReadDoubleQuotedString(this CompilationContext context) => Expression.Call(context.Scanner(), Scanner_ReadDoubleQuotedString);
+    public static MethodCallExpression ReadBacktickString(this CompilationContext context) => Expression.Call(context.Scanner(), Scanner_ReadBacktickString);
+    public static MethodCallExpression ReadCustomString(this CompilationContext context, Expression expectedChars) => Expression.Call(context.Scanner(), Scanner_ReadCustomString, expectedChars);
     public static MethodCallExpression ReadQuotedString(this CompilationContext context) => Expression.Call(context.Scanner(), Scanner_ReadQuotedString);
     public static MethodCallExpression ReadChar(this CompilationContext context, char c) => Expression.Call(context.Scanner(), Scanner_ReadChar, Expression.Constant(c));
 

src/Parlot/Fluent/StringLiteral.cs

@@ -10,7 +10,9 @@ public enum StringLiteralQuotes
 {
     Single,
     Double,
-    SingleOrDouble
+    Backtick,
+    SingleOrDouble,
+    Custom
 }
 
 public sealed class StringLiteral : Parser<TextSpan>, ICompilable, ISeekable
@@ -19,6 +21,7 @@ public sealed class StringLiteral : Parser<TextSpan>, ICompilable, ISeekable
 
     static readonly char[] SingleQuotes = ['\''];
     static readonly char[] DoubleQuotes = ['\"'];
+    static readonly char[] Backtick = ['`'];
     static readonly char[] SingleOrDoubleQuotes = ['\'', '\"'];
 
     private readonly StringLiteralQuotes _quotes;
@@ -31,13 +34,29 @@ public StringLiteral(StringLiteralQuotes quotes)
         {
             StringLiteralQuotes.Single => SingleQuotes,
             StringLiteralQuotes.Double => DoubleQuotes,
+            StringLiteralQuotes.Backtick => Backtick,
             StringLiteralQuotes.SingleOrDouble => SingleOrDoubleQuotes,
-            _ => []
+            _ => throw new InvalidOperationException()
         };
 
         Name = "StringLiteral";
     }
 
+    public StringLiteral(char quote)
+    {
+        _quotes = quote switch
+        {
+            '\'' => StringLiteralQuotes.Single,
+            '\"' => StringLiteralQuotes.Double,
+            '`' => StringLiteralQuotes.Backtick,
+            _ => StringLiteralQuotes.Custom,
+        };
+
+        ExpectedChars = [quote];
+
+        Name = "StringLiteral";
+    }
+
     public bool CanSeek { get; } = true;
 
     public char[] ExpectedChars { get; }
@@ -55,6 +74,8 @@ public override bool Parse(ParseContext context, ref ParseResult<TextSpan> resul
             StringLiteralQuotes.Single => context.Scanner.ReadSingleQuotedString(),
             StringLiteralQuotes.Double => context.Scanner.ReadDoubleQuotedString(),
             StringLiteralQuotes.SingleOrDouble => context.Scanner.ReadQuotedString(),
+            StringLiteralQuotes.Backtick => context.Scanner.ReadBacktickString(),
+            StringLiteralQuotes.Custom => context.Scanner.ReadQuotedString(ExpectedChars),
             _ => false
         };
 
@@ -93,6 +114,8 @@ public CompilationResult Compile(CompilationContext context)
             StringLiteralQuotes.Single => context.ReadSingleQuotedString(),
             StringLiteralQuotes.Double => context.ReadDoubleQuotedString(),
             StringLiteralQuotes.SingleOrDouble => context.ReadQuotedString(),
+            StringLiteralQuotes.Backtick => context.ReadBacktickString(),
+            StringLiteralQuotes.Custom => context.ReadCustomString(Expression.Constant(ExpectedChars)),
             _ => throw new InvalidOperationException()
         };
 

src/Parlot/Scanner.cs

@@ -1,5 +1,7 @@
 using System;
 using Parlot.Fluent;
+using System.Linq;
+
 #if NET8_0_OR_GREATER
 using System.Buffers;
 #endif
@@ -495,14 +497,27 @@ public bool ReadDoubleQuotedString(out ReadOnlySpan<char> result)
         return ReadQuotedString('\"', out result);
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public bool ReadBacktickString() => ReadBacktickString(out _);
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public bool ReadBacktickString(out ReadOnlySpan<char> result)
+    {
+        return ReadQuotedString('`', out result);
+    }
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public bool ReadQuotedString() => ReadQuotedString(out _);
 
-    public bool ReadQuotedString(out ReadOnlySpan<char> result)
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public bool ReadQuotedString(char[] quoteChar) => ReadQuotedString(quoteChar, out _);
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public bool ReadQuotedString(char[] quoteChar, out ReadOnlySpan<char> result)
     {
         var startChar = Cursor.Current;
 
-        if (startChar is not '\'' and not '\"')
+        if (!quoteChar.Contains( startChar ))
         {
             result = [];
             return false;
@@ -511,14 +526,16 @@ public bool ReadQuotedString(out ReadOnlySpan<char> result)
         return ReadQuotedString(startChar, out result);
     }
 
+    public bool ReadQuotedString(out ReadOnlySpan<char> result) => ReadQuotedString(['\'', '\"'],out result);
+
     /// <summary>
-    /// Reads a string token enclosed in single or double quotes.
+    /// Reads a string token enclosed in quotes or custom characters.
     /// </summary>
     /// <remarks>
     /// This method doesn't escape the string, but only validates its content is syntactically correct.
     /// The resulting Span contains the original quotes.
     /// </remarks>
-    private bool ReadQuotedString(char quoteChar, out ReadOnlySpan<char> result)
+    public bool ReadQuotedString(char quoteChar, out ReadOnlySpan<char> result)
     {
         var startChar = Cursor.Current;
         var start = Cursor.Position;

test/Parlot.Tests/CompileTests.cs

@@ -70,6 +70,26 @@ public void ShouldCompileLiteralsWithoutSkipWhiteSpace()
         Assert.Null(result);
     }
 
+    [Fact]
+    public void ShouldCompileCustomStringLiterals()
+    {
+        var parser = new StringLiteral('|').Compile();
+
+        var result = parser.Parse("|hello world|");
+
+        Assert.Equal("hello world", result);
+    }
+
+    [Fact]
+    public void ShouldCompileCustomBacktickStringLiterals()
+    {
+        var parser = new StringLiteral(StringLiteralQuotes.Backtick).Compile();
+
+        var result = parser.Parse("`hello world`");
+
+        Assert.Equal("hello world", result);
+    }
+
     [Fact]
     public void ShouldCompileOrs()
     {

test/Parlot.Tests/ScannerTests.cs

@@ -31,6 +31,19 @@ public void ShouldReadEscapedStringWithMatchingQuotes(string text, string expect
         Assert.Equal(expected, result);
     }
 
+    [Theory]
+    [InlineData('`', "`Lorem ipsum`", "`Lorem ipsum`")]
+    [InlineData('|', "|Lorem ipsum|", "|Lorem ipsum|")]
+    [InlineData('\'', "'Lorem ipsum'", "'Lorem ipsum'")]
+    [InlineData('"', "\"Lorem ipsum\"", "\"Lorem ipsum\"")]
+    public void ShouldReadEscapedStringWithCustomMatchingQuotes(char quote, string text, string expected)
+    {
+        Scanner s = new(text);
+        var success = s.ReadQuotedString([quote], out var result);
+        Assert.True(success);
+        Assert.Equal(expected, result);
+    }
+
     [Theory]
     [InlineData("'Lorem \\n ipsum'", "'Lorem \\n ipsum'")]
     [InlineData("\"Lorem \\n ipsum\"", "\"Lorem \\n ipsum\"")]
@@ -47,6 +60,21 @@ public void ShouldReadStringWithEscapes(string text, string expected)
         Assert.Equal(expected, result);
     }
 
+    [Theory]
+    [InlineData('`', "`Lorem \\n ipsum`", "`Lorem \\n ipsum`")]
+    [InlineData('`', "`Lo\\trem \\n ipsum`", "`Lo\\trem \\n ipsum`")]
+    [InlineData('`', "`Lorem \\u1234 ipsum`", "`Lorem \\u1234 ipsum`")]
+    [InlineData('`', "`Lorem \\xabcd ipsum`", "`Lorem \\xabcd ipsum`")]
+    [InlineData('`', "`\\a ding`", "`\\a ding`")]
+    [InlineData('`', "`Lorem ipsum` \\xabcd", "`Lorem ipsum`")]
+    public void ShouldReadCustomStringWithEscapes(char quote, string text, string expected)
+    {
+        Scanner s = new(text);
+        var success = s.ReadQuotedString([quote], out var result);
+        Assert.True(success);
+        Assert.Equal(expected, result);
+    }
+
     [Theory]
     [InlineData("'Lorem \\w ipsum'")]
     [InlineData("'Lorem \\u12 ipsum'")]
@@ -218,6 +246,20 @@ public void ReadDoubleQuotedStringShouldReadDoubleQuotedStrings()
         Assert.False(new Scanner("\"ab\\\"cd").ReadDoubleQuotedString());
     }
 
+    [Fact]
+    public void ReadBacktickStringShouldBacktickQuotedStrings()
+    {
+        new Scanner("`abcd`").ReadBacktickString(out var result);
+        Assert.Equal("`abcd`", result);
+
+        new Scanner("`a\\nb`").ReadBacktickString(out result);
+        Assert.Equal("`a\\nb`", result);
+
+        Assert.False(new Scanner("`abcd").ReadBacktickString());
+        Assert.False(new Scanner("abcd`").ReadBacktickString());
+        Assert.False(new Scanner("`ab\\`cd").ReadBacktickString());
+    }
+
     [Theory]
     [InlineData("1", "1")]
     [InlineData("123", "123")]