Support integer literals of different radices and decimal numeric literals

shared/src/main/scala/mlscript/NewLexer.scala

@@ -23,6 +23,12 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) {
     c.isLetter || c === '_' || c === '\''
   def isIdentChar(c: Char): Bool =
     isIdentFirstChar(c) || isDigit(c) || c === '\''
+  def isHexDigit(c: Char): Bool =
+    isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
+  def isOctDigit(c: Char): Bool =
+    c >= '0' && c <= '7'
+  def isBinDigit(c: Char): Bool =
+    c === '0' || c === '1'
   def isDigit(c: Char): Bool =
     c >= '0' && c <= '9'
 
@@ -59,15 +65,109 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) {
     if (i < length && pred(bytes(i))) takeWhile(i + 1, bytes(i) :: cur)(pred)
     else (cur.reverseIterator.mkString, i)
 
+  final def num(i: Int): (Lit, Int) = {
+    def test(i: Int, p: Char => Bool): Bool = i < length && p(bytes(i))
+    def zero: IntLit = IntLit(BigInt(0))
+    /** Take a sequence of digits interleaved with underscores. */
+    def takeDigits(i: Int, pred: Char => Bool): (Opt[Str], Int) = {
+      @tailrec def rec(i: Int, acc: Ls[Char], firstSep: Bool, lastSep: Bool): (Str, Bool, Bool, Int) =
+        if (i < length) {
+          val c = bytes(i)
+          if (pred(c)) rec(i + 1, c :: acc, firstSep, false)
+          else if (c === '_') rec(i + 1, acc, acc.isEmpty, true)
+          else (acc.reverseIterator.mkString, firstSep, lastSep, i)
+        }
+        else (acc.reverseIterator.mkString, firstSep, lastSep, i)
+      val (str, firstSep, lastSep, j) = rec(i, Nil, false, false)
+      if (firstSep)
+        raise(WarningReport(
+          msg"Leading separator is not allowed" -> S(loc(i - 1, i)) :: Nil,
+          newDefs = true, source = Lexing))
+      if (lastSep)
+        raise(WarningReport(
+          msg"Trailing separator is not allowed" -> S(loc(j - 1, j)) :: Nil,
+          newDefs = true, source = Lexing))
+      (if (str.isEmpty) N else S(str), j)
+    }
+    /** Take an integer and coverts to `BigInt`. Also checks if it is empty. */
+    def integer(i: Int, radix: Int, desc: Str, pred: Char => Bool): (IntLit, Int) = {
+      takeDigits(i, pred) match {
+        case (N, j) =>
+          raise(ErrorReport(msg"Expect at least one $desc digit" -> S(loc(i, i + 2)) :: Nil,
+            newDefs = true, source = Lexing))
+          (zero, j)
+        case (S(str), j) => (IntLit(BigInt(str, radix)), j)
+      }
+    }
+    def isDecimalStart(ch: Char) = ch === '.' || ch === 'e' || ch === 'E'
+    /** Take a fraction part with an optional exponent part. Call at periods. */
+    def decimal(i: Int, integral: Str): (DecLit, Int) = {
+      val (fraction, j) = if (test(i, _ === '.')) {
+        takeDigits(i + 1, isDigit) match {
+          case (N, j) =>
+            raise(ErrorReport(msg"Expect at least one digit after the decimal point" -> S(loc(i + 1, i + 2)) :: Nil,
+              newDefs = true, source = Lexing))
+            ("", j)
+          case (S(digits), j) => ("." + digits, j)
+        }
+      } else ("", i)
+      val (exponent, k) = if (test(j, ch => ch === 'e' || ch === 'E')) {
+        val (sign, k) = if (test(j + 1, ch => ch === '+' || ch === '-')) {
+          (bytes(j + 1), j + 2)
+        } else {
+          ('+', j + 1)
+        }
+        takeDigits(k, isDigit) match {
+          case (N, l) =>
+            raise(ErrorReport(msg"Expect at least one digit after the exponent sign" -> S(loc(l - 1, l)) :: Nil,
+              newDefs = true, source = Lexing))
+            ("", l)
+          case (S(digits), l) => ("E" + sign + digits, l)
+        }
+      } else {
+        ("", j)
+      }
+      (DecLit(BigDecimal(integral + fraction + exponent)), k)
+    }
+    if (i < length) {
+      bytes(i) match {
+        case '0' if i + 1 < length => bytes(i + 1) match {
+          case 'x' => integer(i + 2, 16, "hexadecimal", isHexDigit)
+          case 'o' => integer(i + 2, 8, "octal", isOctDigit)
+          case 'b' => integer(i + 2, 2, "binary", isBinDigit)
+          case '.' | 'E' | 'e' => decimal(i + 1, "0")
+          case _ => integer(i, 10, "decimal", isDigit)
+        }
+        case '0' => (zero, i + 1)
+        case _ => takeDigits(i, isDigit) match {
+          case (N, j) =>
+            raise(ErrorReport(msg"Expect a numeric literal" -> S(loc(i, i + 1)) :: Nil,
+              newDefs = true, source = Lexing))
+            (zero, i)
+          case (S(integral), j) =>
+            if (j < length && isDecimalStart(bytes(j))) decimal(j, integral)
+            else (IntLit(BigInt(integral)), j)
+        }
+      }
+    } else {
+      raise(ErrorReport(msg"Expect a numeric literal instead of end of input" -> S(loc(i, i + 1)) :: Nil,
+        newDefs = true, source = Lexing))
+      (zero, i)
+    }
+  }
+
   @tailrec final
   def str(i: Int, escapeMode: Bool, cur: Ls[Char] = Nil): (Str, Int) =
     if (escapeMode)
       if (i < length)
         bytes(i) match {
+          case '\\' => str(i + 1, false, '\\' :: cur)
           case '"' => str(i + 1, false, '"' :: cur)
           case 'n' => str(i + 1, false, '\n' :: cur)
           case 't' => str(i + 1, false, '\t' :: cur)
           case 'r' => str(i + 1, false, '\r' :: cur)
+          case 'b' => str(i + 1, false, '\b' :: cur)
+          case 'f' => str(i + 1, false, '\f' :: cur)
           case ch =>
             raise(WarningReport(msg"Found invalid escape character" -> S(loc(i, i + 1)) :: Nil,
               newDefs = true, source = Lexing))
@@ -190,9 +290,9 @@ class NewLexer(origin: Origin, raise: Diagnostic => Unit, dbg: Bool) {
         // else go(j, if (isSymKeyword.contains(n)) KEYWORD(n) else IDENT(n, true))
         else lex(j, ind, next(j, if (isSymKeyword.contains(n)) KEYWORD(n) else IDENT(n, true)))
       case _ if isDigit(c) =>
-        val (str, j) = takeWhile(i)(isDigit)
+        val (lit, j) = num(i)
         // go(j, LITVAL(IntLit(BigInt(str))))
-        lex(j, ind, next(j, LITVAL(IntLit(BigInt(str)))))
+        lex(j, ind, next(j, LITVAL(lit)))
       case _ =>
         pe(msg"unexpected character '${escapeChar(c)}'")
         // go(i + 1, ERROR)

shared/src/test/diff/nu/DecLit.mls

@@ -0,0 +1,139 @@
+:NewDefs
+
+// Real Numbers
+// ============
+
+[0.5, 1.0, 3.14159]
+//│ [0.5, 1.0, 3.14159]
+//│ res
+//│     = [ 0.5, 1, 3.14159 ]
+
+[1e100, 1E100, 1e+100, 1E+100, 1E-11, 1e-10, 1E-2, 1e-2]
+//│ [1E+100, 1E+100, 1E+100, 1E+100, 1E-11, 1E-10, 0.01, 0.01]
+//│ res
+//│     = [
+//│         1e+100, 1e+100,
+//│         1e+100, 1e+100,
+//│          1e-11,  1e-10,
+//│           0.01,   0.01
+//│       ]
+
+[3.14e-10, 3.14E-10, 3.14e+10, 3.14E+10]
+//│ [3.14E-10, 3.14E-10, 3.14E+10, 3.14E+10]
+//│ res
+//│     = [ 3.14e-10, 3.14e-10, 31400000000, 31400000000 ]
+
+[0.5e-10, 0.5E-10, 0.5e+10, 0.5E+10]
+//│ [5E-11, 5E-11, 5E+9, 5E+9]
+//│ res
+//│     = [ 5e-11, 5e-11, 5000000000, 5000000000 ]
+
+// Separators in integral, fractional, and exponent parts.
+[12_34_56.0, 12_34_56.78_90]
+[1_2.3_4e-1_0, 1_2.3_4e+1_0, 1_2.3_4e1_0]
+[1_2.3_4E-1_0, 1_2.3_4E+1_0, 1_2.3_4E1_0]
+//│ [1.234E-9, 1.234E+11, 1.234E+11]
+//│ res
+//│     = [ 123456, 123456.789 ]
+//│ res
+//│     = [ 1.234e-9, 123400000000, 123400000000 ]
+//│ res
+//│     = [ 1.234e-9, 123400000000, 123400000000 ]
+
+// Conflict with tuple index selection.
+:pe
+.1
+//│ ╔══[PARSE ERROR] Unexpected selector in expression position
+//│ ║  l.45: 	.1
+//│ ╙──      	^^
+//│ ╔══[PARSE ERROR] Unexpected end of input; an expression was expected here
+//│ ║  l.45: 	.1
+//│ ╙──      	  ^
+//│ ()
+//│ res
+//│     = undefined
+
+// Corner cases.
+:pe
+0.E10
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the decimal point
+//│ ║  l.58: 	0.E10
+//│ ╙──      	  ^
+//│ 0E+10
+//│ res
+//│     = 0
+
+:pe
+0.0E
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the exponent sign
+//│ ║  l.67: 	0.0E
+//│ ╙──      	   ^
+//│ 0.0
+//│ res
+//│     = 0
+
+:pe
+0.0E+
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the exponent sign
+//│ ║  l.76: 	0.0E+
+//│ ╙──      	    ^
+//│ 0.0
+//│ res
+//│     = 0
+
+:pe
+0E
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the exponent sign
+//│ ║  l.85: 	0E
+//│ ╙──      	 ^
+//│ 0
+//│ res
+//│     = 0
+
+:pe
+0E+
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the exponent sign
+//│ ║  l.94: 	0E+
+//│ ╙──      	  ^
+//│ 0
+//│ res
+//│     = 0
+
+:pe
+1234E
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the exponent sign
+//│ ║  l.103: 	1234E
+//│ ╙──       	    ^
+//│ 1234
+//│ res
+//│     = 1234
+
+:pe
+4378.
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the decimal point
+//│ ║  l.112: 	4378.
+//│ ╙──       	     ^
+//│ 4378
+//│ res
+//│     = 4378
+
+:pe
+5.
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the decimal point
+//│ ║  l.121: 	5.
+//│ ╙──       	  ^
+//│ 5
+//│ res
+//│     = 5
+
+:pe
+789.E
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the decimal point
+//│ ║  l.130: 	789.E
+//│ ╙──       	    ^
+//│ ╔══[LEXICAL ERROR] Expect at least one digit after the exponent sign
+//│ ║  l.130: 	789.E
+//│ ╙──       	    ^
+//│ 789
+//│ res
+//│     = 789