Add support for ES2023: Hashbang comments (#44)

lydell · Feb 8, 2024 · 7cd8a56 · 7cd8a56
1 parent b734e90
commit 7cd8a56
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -63,6 +63,7 @@ type Token =
   | { type: "RegularExpressionLiteral"; value: string; closed: boolean }
   | { type: "MultiLineComment"; value: string; closed: boolean }
   | { type: "SingleLineComment"; value: string }
+  | { type: "HashbangComment"; value: string }
   | { type: "IdentifierName"; value: string }
   | { type: "PrivateIdentifier"; value: string }
   | { type: "NumericLiteral"; value: string }
@@ -178,6 +179,21 @@ Examples:
 //
 ```
 
+### HashbangComment
+
+_Spec: [HashbangComment]_
+
+Note that a HashbangComment can only occur at the very start of the string that is being tokenized. Anywhere else you will likely get an Invalid token `#` followed by a Punctuator token `!`.
+
+Examples:
+
+<!-- prettier-ignore -->
+```js
+#!/usr/bin/env node
+#! console.log("commented", out + code);
+#!
+```
+
 ### IdentifierName
 
 _Spec: [IdentifierName]_
@@ -419,7 +435,7 @@ All possible values in JSX children:
 
 The intention is to always support the latest ECMAScript version whose feature set has been finalized.
 
-Currently, ECMAScript 2022 is supported.
+Currently, ECMAScript 2023 is supported.
 
 #### Annex B and C (strict mode)
 
@@ -641,6 +657,7 @@ See [benchmark.js] if you want to run benchmarks yourself.
 [divpunctuator]: https://tc39.es/ecma262/#prod-DivPunctuator
 [ecmascript language: lexical grammar]: https://tc39.es/ecma262/#sec-ecmascript-language-lexical-grammar
 [example.test.js]: test/example.test.js
+[hashbangcomment]: https://tc39.es/ecma262/#prod-HashbangComment
 [identifiername]: https://tc39.es/ecma262/#prod-IdentifierName
 [identifierpart]: https://tc39.es/ecma262/#prod-IdentifierPart
 [jsx specification]: https://facebook.github.io/jsx/

diff --git a/index.coffee b/index.coffee
@@ -147,6 +147,10 @@ SingleLineComment = ///
   //.*
 ///y
 
+HashbangComment = ///
+  ^#!.*
+///
+
 JSXPunctuator = ///
   [ < > . : = { } ]
   |
@@ -215,6 +219,13 @@ module.exports = jsTokens = (input, {jsx = false} = {}) ->
   parenNesting = 0
   postfixIncDec = false
 
+  if match = HashbangComment.exec(input)
+    yield {
+      type: "HashbangComment",
+      value: match[0],
+    }
+    lastIndex = match[0].length
+
   while lastIndex < length
     mode = stack[stack.length - 1]
 

diff --git a/index.d.ts b/index.d.ts
@@ -7,6 +7,7 @@ export declare type Token =
   | { type: "RegularExpressionLiteral"; value: string; closed: boolean }
   | { type: "MultiLineComment"; value: string; closed: boolean }
   | { type: "SingleLineComment"; value: string }
+  | { type: "HashbangComment"; value: string }
   | { type: "IdentifierName"; value: string }
   | { type: "PrivateIdentifier"; value: string }
   | { type: "NumericLiteral"; value: string }

diff --git a/index.test-d.ts b/index.test-d.ts
@@ -57,6 +57,10 @@ const tokens: Array<string> = Array.from(jsTokens(""), (token) => {
       expectError(token.closed);
       return token.value;
 
+    case "HashbangComment":
+      expectError(token.closed);
+      return token.value;
+
     case "IdentifierName":
       expectError(token.closed);
       return token.value;
@@ -125,6 +129,10 @@ const jsxTokens: Array<string> = Array.from(
         expectError(token.closed);
         return token.value;
 
+      case "HashbangComment":
+        expectError(token.closed);
+        return token.value;
+
       case "IdentifierName":
         expectError(token.closed);
         return token.value;

diff --git a/test/example.test.js b/test/example.test.js
@@ -4,12 +4,21 @@ const jsTokens = require("../build/index");
 
 test("all tokens", () => {
   const code =
-    'console.log("", ``, `a${1}b${this.#private}`, /**/ /./, 0x1Fn) //\r\n#\'';
+    '#!hashbang\nconsole.log("", ``, `a${1}b${this.#private}`, /**/ /./, 0x1Fn) //\r\n#!\'';
 
   const tokens = Array.from(jsTokens(code));
 
   expect(tokens).toMatchInlineSnapshot(`
     [
+      {
+        "type": "HashbangComment",
+        "value": "#!hashbang",
+      },
+      {
+        "type": "LineTerminatorSequence",
+        "value": "
+    ",
+      },
       {
         "type": "IdentifierName",
         "value": "console",
@@ -136,6 +145,10 @@ test("all tokens", () => {
         "type": "Invalid",
         "value": "#",
       },
+      {
+        "type": "Punctuator",
+        "value": "!",
+      },
       {
         "closed": false,
         "type": "StringLiteral",

diff --git a/test/index.test.js b/test/index.test.js
@@ -178,6 +178,24 @@ describe("Token", () => {
     match("//comment\t\n", "//comment\t");
   });
 
+  testToken("HashbangComment", [], (match) => {
+    match("#!");
+    match("#!comment");
+    match("#! comment");
+    match("#!comment ");
+    match("#!comment\n", "#!comment");
+    match("#!comment\r", "#!comment");
+    match("#!comment\u2028", "#!comment");
+    match("#!comment\u2029", "#!comment");
+    match("#!comment\r\n", "#!comment");
+    match("#!comment \n", "#!comment ");
+    match("#!comment\t\n", "#!comment\t");
+    match("#!comment\ncode", ["#!comment", "\n", "code"]);
+    match("#!comment\r\ncode", ["#!comment", "\r\n", "code"]);
+    match(" #!", false);
+    match("\n#!", false);
+  });
+
   testToken("MultiLineComment", [], (match) => {
     match("/**/");
     match("/*comment*/");