Replace tokenType with quoted:boolean field in IdentifierNode

Better to limit the amount of lexer information we expose in parser result. Also the code in parser is more likely to be updated when lexer tokens change, unlike the code in formatter, which is further separated.
sql-formatter-org · Nov 13, 2023 · a0811e9 · a0811e9
1 parent 282b84d
commit a0811e9
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 32 deletions.
diff --git a/src/formatter/ExpressionFormatter.ts b/src/formatter/ExpressionFormatter.ts
@@ -510,7 +510,9 @@ export default class ExpressionFormatter {
   }
 
   private showIdentifier(node: IdentifierNode): string {
-    if (node.tokenType === TokenType.IDENTIFIER || node.tokenType === TokenType.ARRAY_IDENTIFIER) {
+    if (node.quoted) {
+      return node.text;
+    } else {
       switch (this.cfg.identifierCase) {
         case 'preserve':
           return node.text;
@@ -519,8 +521,6 @@ export default class ExpressionFormatter {
         case 'lower':
           return node.text.toLowerCase();
       }
-    } else {
-      return node.text;
     }
   }
 }
diff --git a/src/parser/ast.ts b/src/parser/ast.ts
@@ -125,7 +125,7 @@ export interface PropertyAccessNode extends BaseNode {
 
 export interface IdentifierNode extends BaseNode {
   type: NodeType.identifier;
-  tokenType: TokenType;
+  quoted: boolean;
   text: string;
 }
 

diff --git a/src/parser/grammar.ne b/src/parser/grammar.ne
@@ -16,12 +16,6 @@ const lexer = new LexerAdapter(chunk => []);
 // which otherwise produce single element nested inside two arrays
 const unwrap = <T>([[el]]: T[][]): T => el;
 
-const toIdentifierNode = (token: Token): IdentifierNode => ({
-  type: NodeType.identifier,
-  tokenType: token.type,
-  text: token.text,
-});
-
 const toKeywordNode = (token: Token): KeywordNode => ({
   type: NodeType.keyword,
   tokenType: token.type,
@@ -208,7 +202,7 @@ atomic_expression ->
 array_subscript -> %ARRAY_IDENTIFIER _ square_brackets {%
   ([arrayToken, _, brackets]) => ({
     type: NodeType.array_subscript,
-    array: addComments({ type: NodeType.identifier, tokenType: TokenType.ARRAY_IDENTIFIER, text: arrayToken.text}, { trailing: _ }),
+    array: addComments({ type: NodeType.identifier, quoted: false, text: arrayToken.text}, { trailing: _ }),
     parenthesis: brackets,
   })
 %}
@@ -315,7 +309,7 @@ operator -> ( %OPERATOR ) {% ([[token]]) => ({ type: NodeType.operator, text: to
 identifier ->
   ( %IDENTIFIER
   | %QUOTED_IDENTIFIER
-  | %VARIABLE ) {% ([[token]]) => toIdentifierNode(token) %}
+  | %VARIABLE ) {% ([[token]]) => ({ type: NodeType.identifier, quoted: token.type !== "IDENTIFIER", text: token.text }) %}
 
 parameter ->
   ( %NAMED_PARAMETER

diff --git a/test/unit/__snapshots__/Parser.test.ts.snap b/test/unit/__snapshots__/Parser.test.ts.snap
@@ -7,8 +7,8 @@ Array [
       Object {
         "children": Array [
           Object {
+            "quoted": false,
             "text": "age",
-            "tokenType": "IDENTIFIER",
             "type": "identifier",
           },
           Object {
@@ -127,8 +127,8 @@ Array [
             },
             "expr": Array [
               Object {
+                "quoted": false,
                 "text": "foo",
-                "tokenType": "IDENTIFIER",
                 "type": "identifier",
               },
             ],
@@ -289,8 +289,8 @@ Array [
         "children": Array [
           Object {
             "object": Object {
+              "quoted": false,
               "text": "ident",
-              "tokenType": "IDENTIFIER",
               "type": "identifier",
             },
             "property": Object {
@@ -322,8 +322,8 @@ Array [
         "children": Array [
           Object {
             "array": Object {
+              "quoted": false,
               "text": "my_array",
-              "tokenType": "ARRAY_IDENTIFIER",
               "type": "identifier",
             },
             "parenthesis": Object {
@@ -363,8 +363,8 @@ Array [
         "children": Array [
           Object {
             "array": Object {
+              "quoted": false,
               "text": "my_array",
-              "tokenType": "ARRAY_IDENTIFIER",
               "trailingComments": Array [
                 Object {
                   "precedingWhitespace": " ",
@@ -412,17 +412,17 @@ Array [
           Object {
             "children": Array [
               Object {
+                "quoted": false,
                 "text": "foo",
-                "tokenType": "IDENTIFIER",
                 "type": "identifier",
               },
               Object {
                 "text": ":",
                 "type": "operator",
               },
               Object {
+                "quoted": false,
                 "text": "bar",
-                "tokenType": "IDENTIFIER",
                 "type": "identifier",
               },
             ],
@@ -501,8 +501,8 @@ Array [
             "type": "keyword",
           },
           Object {
+            "quoted": false,
             "text": "a",
-            "tokenType": "IDENTIFIER",
             "type": "identifier",
           },
           Object {
@@ -524,8 +524,8 @@ Array [
             "type": "function_call",
           },
           Object {
+            "quoted": false,
             "text": "b",
-            "tokenType": "IDENTIFIER",
             "type": "identifier",
           },
         ],
@@ -549,8 +549,8 @@ Array [
   Object {
     "children": Array [
       Object {
+        "quoted": false,
         "text": "foo",
-        "tokenType": "IDENTIFIER",
         "type": "identifier",
       },
     ],
@@ -560,8 +560,8 @@ Array [
   Object {
     "children": Array [
       Object {
+        "quoted": false,
         "text": "bar",
-        "tokenType": "IDENTIFIER",
         "type": "identifier",
       },
     ],
@@ -580,8 +580,8 @@ Array [
           Object {
             "children": Array [
               Object {
+                "quoted": false,
                 "text": "birth_year",
-                "tokenType": "IDENTIFIER",
                 "type": "identifier",
               },
               Object {
@@ -591,8 +591,8 @@ Array [
               Object {
                 "children": Array [
                   Object {
+                    "quoted": false,
                     "text": "CURRENT_DATE",
-                    "tokenType": "IDENTIFIER",
                     "type": "identifier",
                   },
                   Object {
@@ -638,20 +638,20 @@ Array [
           Object {
             "object": Object {
               "object": Object {
+                "quoted": false,
                 "text": "foo",
-                "tokenType": "IDENTIFIER",
                 "type": "identifier",
               },
               "property": Object {
+                "quoted": false,
                 "text": "bar",
-                "tokenType": "IDENTIFIER",
                 "type": "identifier",
               },
               "type": "property_access",
             },
             "property": Object {
+              "quoted": false,
               "text": "baz",
-              "tokenType": "IDENTIFIER",
               "type": "identifier",
             },
             "type": "property_access",
@@ -679,8 +679,8 @@ Array [
       Object {
         "children": Array [
           Object {
+            "quoted": false,
             "text": "foo",
-            "tokenType": "IDENTIFIER",
             "type": "identifier",
           },
         ],
@@ -695,8 +695,8 @@ Array [
       Object {
         "children": Array [
           Object {
+            "quoted": false,
             "text": "bar",
-            "tokenType": "IDENTIFIER",
             "type": "identifier",
           },
         ],
@@ -721,8 +721,8 @@ Array [
       Object {
         "children": Array [
           Object {
+            "quoted": false,
             "text": "foo",
-            "tokenType": "IDENTIFIER",
             "type": "identifier",
           },
         ],
@@ -737,8 +737,8 @@ Array [
       Object {
         "children": Array [
           Object {
+            "quoted": false,
             "text": "baz",
-            "tokenType": "IDENTIFIER",
             "type": "identifier",
           },
         ],