From 30cd0e49f76f2eee8ab37c4c48b01aeb8f9f9736 Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Sun, 10 Jul 2022 03:23:37 +0900 Subject: [PATCH 1/7] JavaScript: extract variables and constants in destructuring assignment Close #1112. Signed-off-by: Masatake YAMATO --- .../js-destructural-binding.d/args.ctags | 1 + .../js-destructural-binding.d/expected.tags | 59 +++++++ .../js-destructural-binding.d/input.js | 56 ++++++ .../js-destructural-binding.d/validator | 1 + .../js-empty-class-name.d/expected.tags | 1 + parsers/jscript.c | 160 +++++++++++++++++- 6 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/args.ctags create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/expected.tags create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/input.js create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/validator diff --git a/Units/parser-javascript.r/js-destructural-binding.d/args.ctags b/Units/parser-javascript.r/js-destructural-binding.d/args.ctags new file mode 100644 index 0000000000..5ee5f79f70 --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/args.ctags @@ -0,0 +1 @@ +--sort=no diff --git a/Units/parser-javascript.r/js-destructural-binding.d/expected.tags b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags new file mode 100644 index 0000000000..b77bd2e284 --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags @@ -0,0 +1,59 @@ +x input.js /^const [x] = [1];$/;" C +y input.js /^const [y, z] = [1, 2, 3, 4, 5];$/;" C +z input.js /^const [y, z] = [1, 2, 3, 4, 5];$/;" C +a input.js /^let [a=5, b=7] = [1];$/;" v +b input.js /^let [a=5, b=7] = [1];$/;" v +c input.js /^let [c, , d] = [1, 2, 3];$/;" v +d input.js /^let [c, , d] = [1, 2, 3];$/;" v +e input.js /^let [e, f = 0, , g] = [1, 2, 3, 4];$/;" v +f input.js /^let [e, f = 0, , g] = [1, 2, 3, 4];$/;" v +g input.js /^let [e, f = 0, , g] = [1, 2, 3, 4];$/;" v +h input.js /^const [h, i, ...[j, k]] = [1, 2, 3, 4];$/;" C +i input.js /^const [h, i, ...[j, k]] = [1, 2, 3, 4];$/;" C +j input.js /^const [h, i, ...[j, k]] = [1, 2, 3, 4];$/;" C +k input.js /^const [h, i, ...[j, k]] = [1, 2, 3, 4];$/;" C +l input.js /^const [l, m, ...[n, o, ...[p, q]]] = [1, 2, 3, 4, 5, 6];$/;" C +m input.js /^const [l, m, ...[n, o, ...[p, q]]] = [1, 2, 3, 4, 5, 6];$/;" C +n input.js /^const [l, m, ...[n, o, ...[p, q]]] = [1, 2, 3, 4, 5, 6];$/;" C +o input.js /^const [l, m, ...[n, o, ...[p, q]]] = [1, 2, 3, 4, 5, 6];$/;" C +p input.js /^const [l, m, ...[n, o, ...[p, q]]] = [1, 2, 3, 4, 5, 6];$/;" C +q input.js /^const [l, m, ...[n, o, ...[p, q]]] = [1, 2, 3, 4, 5, 6];$/;" C +A input.js /^const [A, B, ...{ C, D }] = []$/;" C +B input.js /^const [A, B, ...{ C, D }] = []$/;" C +C input.js /^const [A, B, ...{ C, D }] = []$/;" C +D input.js /^const [A, B, ...{ C, D }] = []$/;" C +E input.js /^ E: 42,$/;" p variable:user +F input.js /^ F: true$/;" p variable:user +user input.js /^const user = {$/;" v +E input.js /^const {E, F} = user;$/;" C +F input.js /^const {E, F} = user;$/;" C +G input.js /^const {E: G, F: H} = user;$/;" C +H input.js /^const {E: G, F: H} = user;$/;" C +I input.js /^const {I = 10, J = 5} = {I: 3};$/;" C +J input.js /^const {I = 10, J = 5} = {I: 3};$/;" C +I input.js /^const {I = 10, J = 5} = {I: 3};$/;" p variable:anonymousObject785a93f40105 +anonymousObject785a93f40105 input.js /^const {I = 10, J = 5} = {I: 3};$/;" v +K input.js /^let {a: K = 10, b: L = 5} = {a: 3};$/;" v +L input.js /^let {a: K = 10, b: L = 5} = {a: 3};$/;" v +a input.js /^let {a: K = 10, b: L = 5} = {a: 3};$/;" p variable:anonymousObject785a93f40205 +anonymousObject785a93f40205 input.js /^let {a: K = 10, b: L = 5} = {a: 3};$/;" v +M input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" v +N input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" v +O input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" v +M input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" p variable:anonymousObject785a93f40305 +N input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" p variable:anonymousObject785a93f40305 +c input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" p variable:anonymousObject785a93f40305 +d input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" p variable:anonymousObject785a93f40305 +anonymousObject785a93f40305 input.js /^let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40}$/;" v +title input.js /^ title: 'Scratchpad',$/;" p variable:metadata +anonymousObject785a93f40405 input.js /^ {$/;" v variable:metadata +locale input.js /^ locale: 'de',$/;" p variable:metadata.anonymousObject785a93f40405 +localization_tags input.js /^ localization_tags: [],$/;" p variable:metadata.anonymousObject785a93f40405 +last_edit input.js /^ last_edit: '2014-04-14T08:43:37',$/;" p variable:metadata.anonymousObject785a93f40405 +url input.js /^ url: '\/de\/docs\/Tools\/Scratchpad',$/;" p variable:metadata.anonymousObject785a93f40405 +title input.js /^ title: 'JavaScript-Umgebung'$/;" p variable:metadata.anonymousObject785a93f40405 +translations input.js /^ translations: [$/;" p variable:metadata +url input.js /^ url: '\/en-US\/docs\/Tools\/Scratchpad'$/;" p variable:metadata +metadata input.js /^const metadata = {$/;" v +englishTitle input.js /^ title: englishTitle, \/\/ rename$/;" v +localeTitle input.js /^ title: localeTitle, \/\/ rename$/;" v diff --git a/Units/parser-javascript.r/js-destructural-binding.d/input.js b/Units/parser-javascript.r/js-destructural-binding.d/input.js new file mode 100644 index 0000000000..5fc18b740f --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/input.js @@ -0,0 +1,56 @@ +// Derrived from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Destructuring_assignment +const [x] = [1]; +const [y, z] = [1, 2, 3, 4, 5]; +let [a=5, b=7] = [1]; + +let [c, , d] = [1, 2, 3]; +let [e, f = 0, , g] = [1, 2, 3, 4]; + +let [,,] = [1, 2, 3]; +let [,] = [1, 2, 3]; +let [] = [1, 2, 3]; + +const [h, i, ...[j, k]] = [1, 2, 3, 4]; +const [l, m, ...[n, o, ...[p, q]]] = [1, 2, 3, 4, 5, 6]; + +const [A, B, ...{ C, D }] = [] + +const user = { + E: 42, + F: true +}; + +const {E, F} = user; + +const {E: G, F: H} = user; + + +const {I = 10, J = 5} = {I: 3}; + +let {a: K = 10, b: L = 5} = {a: 3}; + +let {M, N, ...O} = {M: 10, N: 20, c: 30, d: 40} + + +const metadata = { + title: 'Scratchpad', + translations: [ + { + locale: 'de', + localization_tags: [], + last_edit: '2014-04-14T08:43:37', + url: '/de/docs/Tools/Scratchpad', + title: 'JavaScript-Umgebung' + } + ], + url: '/en-US/docs/Tools/Scratchpad' +}; + +let { + title: englishTitle, // rename + translations: [ + { + title: localeTitle, // rename + }, + ], +} = metadata; diff --git a/Units/parser-javascript.r/js-destructural-binding.d/validator b/Units/parser-javascript.r/js-destructural-binding.d/validator new file mode 100644 index 0000000000..64f5a0a681 --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/validator @@ -0,0 +1 @@ +node diff --git a/Units/parser-javascript.r/js-empty-class-name.d/expected.tags b/Units/parser-javascript.r/js-empty-class-name.d/expected.tags index 438218215f..3a51634fb7 100644 --- a/Units/parser-javascript.r/js-empty-class-name.d/expected.tags +++ b/Units/parser-javascript.r/js-empty-class-name.d/expected.tags @@ -1,2 +1,3 @@ +prop input.js /^var {prop} = { prop: "value" };$/;" v prop input.js /^var {prop} = { prop: "value" };$/;" p variable:anonymousObject4ca5b60a0105 anonymousObject4ca5b60a0105 input.js /^var {prop} = { prop: "value" };$/;" v diff --git a/parsers/jscript.c b/parsers/jscript.c index bf62ea545c..038d8f2c69 100644 --- a/parsers/jscript.c +++ b/parsers/jscript.c @@ -2799,7 +2799,8 @@ static bool parseStatementRHS (tokenInfo *const name, tokenInfo *const token, st if (state->indexForName == CORK_NIL) { - state->indexForName = makeJsTag (name, state->isConst ? JSTAG_CONSTANT : JSTAG_VARIABLE, NULL, NULL); + if (!vStringIsEmpty (name->string)) + state->indexForName = makeJsTag (name, state->isConst ? JSTAG_CONSTANT : JSTAG_VARIABLE, NULL, NULL); if (isType (token, TOKEN_IDENTIFIER)) canbe_arrowfun = true; } @@ -2868,6 +2869,144 @@ static bool parseStatementRHS (tokenInfo *const name, tokenInfo *const token, st return true; } +static bool parseObjectDestructuring (tokenInfo *const token, bool is_const); +static bool parseArrayDestructuring (tokenInfo *const token, bool is_const) +{ + int nest_level = 1; + bool in_left_side = true; + bool found = false; + + while (nest_level > 0 && ! isType (token, TOKEN_EOF)) + { + readToken (token); + if (isType (token, TOKEN_OPEN_SQUARE)) + { + in_left_side = true; + nest_level++; + } + else if (isType (token, TOKEN_CLOSE_SQUARE)) + { + in_left_side = false; + nest_level--; + } + else if (isType (token, TOKEN_OPEN_CURLY)) + { + in_left_side = false; + if (parseObjectDestructuring (token, is_const)) + found = true; + } + else if (isType (token, TOKEN_COMMA) + || isType (token, TOKEN_DOTS)) + in_left_side = true; + else if (in_left_side && isType (token, TOKEN_IDENTIFIER)) + { + in_left_side = false; + makeJsTag (token, + is_const ? JSTAG_CONSTANT : JSTAG_VARIABLE, + NULL, NULL); + found = true; + } + else if (isType (token, TOKEN_EQUAL_SIGN)) + { + in_left_side = false; + /* TODO: SKIP */ + } + else + in_left_side = false; + } + + return found; +} + +static bool parseObjectDestructuring (tokenInfo *const token, bool is_const) +{ + tokenInfo *const name = newToken (); + bool found = false; + + /* + * let { k0: v0, k1: v1 = 0, v3 }; + * | | || | | | | + * ^...|..|^...|..|....^....|.: start + * ....^..|....^..|.........|.: colon + * .......^.......^.........|.: tagged (made a tag for an id after colon) + * .........................^.: BREAK + */ + enum objDestructuringState { + OBJ_DESTRUCTURING_START, + OBJ_DESTRUCTURING_COLON, + OBJ_DESTRUCTURING_TAGGED, + } state = OBJ_DESTRUCTURING_START; + + while (! isType (token, TOKEN_EOF)) + { + readToken (token); + if (isType (token, TOKEN_OPEN_CURLY)) + { + if (parseObjectDestructuring (token, is_const)) + found = true; + if (state == OBJ_DESTRUCTURING_COLON) + state = OBJ_DESTRUCTURING_TAGGED; + } + else if (isType (token, TOKEN_CLOSE_CURLY)) + { + if (!vStringIsEmpty(name->string)) + { + makeJsTag (name, + is_const ? JSTAG_CONSTANT : JSTAG_VARIABLE, + NULL, NULL); + found = true; + } + break; + } + else if (isType (token, TOKEN_OPEN_SQUARE)) + { + if (parseArrayDestructuring (token, is_const)) + found = true; + if (state == OBJ_DESTRUCTURING_COLON) + state = OBJ_DESTRUCTURING_TAGGED; + } + else if (isType (token, TOKEN_IDENTIFIER)) + { + if (state == OBJ_DESTRUCTURING_COLON) + { + makeJsTag (token, + is_const ? JSTAG_CONSTANT : JSTAG_VARIABLE, + NULL, NULL); + found = true; + state = OBJ_DESTRUCTURING_TAGGED; + } + else if (state == OBJ_DESTRUCTURING_START + && vStringIsEmpty(name->string)) + copyToken(name, token, true); + } + else if (isType (token, TOKEN_COMMA)) + { + if (!vStringIsEmpty(name->string)) + { + makeJsTag (name, + is_const ? JSTAG_CONSTANT : JSTAG_VARIABLE, + NULL, NULL); + found = true; + vStringClear (name->string); + } + state = OBJ_DESTRUCTURING_START; + } + else if (isType (token, TOKEN_COLON)) + { + vStringClear (name->string); + state = OBJ_DESTRUCTURING_COLON; + } + else + { + if (state == OBJ_DESTRUCTURING_COLON) + state = OBJ_DESTRUCTURING_TAGGED; + } + } + + deleteToken (name); + return found; +} + static bool parseStatement (tokenInfo *const token, bool is_inside_class) { TRACE_ENTER_TEXT("is_inside_class: %s", is_inside_class? "yes": "no"); @@ -2935,6 +3074,22 @@ static bool parseStatement (tokenInfo *const token, bool is_inside_class) state.isGlobal = true; } readToken(token); + + if (state.isGlobal) + { + bool found = false; + if (isType (token, TOKEN_OPEN_CURLY)) + found = parseObjectDestructuring (token, state.isConst); + else if (isType (token, TOKEN_OPEN_SQUARE)) + found = parseArrayDestructuring (token, state.isConst); + + if (found) + { + /* Adjust the context to the code for non-destructing. */ + found_lhs = true; + readToken(token); + } + } } nextVar: @@ -3025,7 +3180,8 @@ static bool parseStatement (tokenInfo *const token, bool is_inside_class) * Handles this syntax: * var g_var2; */ - state.indexForName = makeJsTag (name, state.isConst ? JSTAG_CONSTANT : JSTAG_VARIABLE, NULL, NULL); + if (!vStringIsEmpty (name->string)) + state.indexForName = makeJsTag (name, state.isConst ? JSTAG_CONSTANT : JSTAG_VARIABLE, NULL, NULL); } /* * Statement has ended. From 2100525bf347dac7071896aa70ada55e26fecaca Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Fri, 13 Jan 2023 06:14:53 +0900 Subject: [PATCH 2/7] JavaScript: add more trace code Signed-off-by: Masatake YAMATO --- parsers/jscript.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/parsers/jscript.c b/parsers/jscript.c index 038d8f2c69..3bb727af9e 100644 --- a/parsers/jscript.c +++ b/parsers/jscript.c @@ -1432,6 +1432,8 @@ static void readToken (tokenInfo *const token) static int parseMethodsInAnonymousObject (tokenInfo *const token) { + TRACE_ENTER(); + int index = CORK_NIL; tokenInfo *const anon_object = newToken (); @@ -1453,11 +1455,14 @@ static int parseMethodsInAnonymousObject (tokenInfo *const token) deleteToken (anon_object); + TRACE_LEAVE(); return index; } static void skipArgumentList (tokenInfo *const token, bool include_newlines, vString *const repr) { + TRACE_ENTER(); + if (isType (token, TOKEN_OPEN_PAREN)) /* arguments? */ { int nest_level = 1; @@ -1486,6 +1491,7 @@ static void skipArgumentList (tokenInfo *const token, bool include_newlines, vSt } readTokenFull (token, include_newlines, NULL); } + TRACE_LEAVE(); } static void skipArrayList (tokenInfo *const token, bool include_newlines) From 783f4d39993b2fc1ecc5fe11b682c9372790960b Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Fri, 13 Jan 2023 06:25:44 +0900 Subject: [PATCH 3/7] JavaScript: don't analyze inside parameter lists Signed-off-by: Masatake YAMATO --- .../js-destructural-binding.d/args.ctags | 1 + .../js-destructural-binding.d/expected.tags | 7 ++ .../js-destructural-binding.d/input-0.js | 13 ++++ .../js-destructural-binding.d/input-1.js | 5 ++ parsers/jscript.c | 72 +++++++++++++------ 5 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/input-0.js create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/input-1.js diff --git a/Units/parser-javascript.r/js-destructural-binding.d/args.ctags b/Units/parser-javascript.r/js-destructural-binding.d/args.ctags index 5ee5f79f70..7311baa956 100644 --- a/Units/parser-javascript.r/js-destructural-binding.d/args.ctags +++ b/Units/parser-javascript.r/js-destructural-binding.d/args.ctags @@ -1 +1,2 @@ --sort=no +--fields=+S diff --git a/Units/parser-javascript.r/js-destructural-binding.d/expected.tags b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags index b77bd2e284..aeb61943da 100644 --- a/Units/parser-javascript.r/js-destructural-binding.d/expected.tags +++ b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags @@ -57,3 +57,10 @@ url input.js /^ url: '\/en-US\/docs\/Tools\/Scratchpad'$/;" p variable:metadata metadata input.js /^const metadata = {$/;" v englishTitle input.js /^ title: englishTitle, \/\/ rename$/;" v localeTitle input.js /^ title: localeTitle, \/\/ rename$/;" v +userDisplayName input-0.js /^function userDisplayName({displayName: dname}) {$/;" f signature:({displayName: dname}) +whois input-0.js /^function whois({displayName, fullName: {firstName: name}}) {$/;" f signature:({displayName, fullName: {firstName: name}}) +drawChart input-0.js /^function drawChart({size = 'big', coords = {x: 0, y: 0}, radius = 25} = {}) {$/;" f signature:({size = 'big', coords = {x: 0, y: 0}, radius = 25} = {}) +f input-1.js /^function f({ u, x }) {$/;" f signature:({ u, x }) +anonymousObjectf91cef720105 input-1.js /^f({u: 1, x: 2})$/;" v +u input-1.js /^f({u: 1, x: 2})$/;" p variable:anonymousObjectf91cef720105 +x input-1.js /^f({u: 1, x: 2})$/;" p variable:anonymousObjectf91cef720105 diff --git a/Units/parser-javascript.r/js-destructural-binding.d/input-0.js b/Units/parser-javascript.r/js-destructural-binding.d/input-0.js new file mode 100644 index 0000000000..dfc9f7fc43 --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/input-0.js @@ -0,0 +1,13 @@ +// Derrived from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Destructuring_assignment +function userDisplayName({displayName: dname}) { + return dname; +} + +function whois({displayName, fullName: {firstName: name}}) { + return `${displayName} is ${name}`; +} + +function drawChart({size = 'big', coords = {x: 0, y: 0}, radius = 25} = {}) { + console.log(size, coords, radius); + // do some chart drawing +} diff --git a/Units/parser-javascript.r/js-destructural-binding.d/input-1.js b/Units/parser-javascript.r/js-destructural-binding.d/input-1.js new file mode 100644 index 0000000000..186ed50077 --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/input-1.js @@ -0,0 +1,5 @@ +function f({ u, x }) { + return (u + x) +} + +f({u: 1, x: 2}) diff --git a/parsers/jscript.c b/parsers/jscript.c index 3bb727af9e..e937d280da 100644 --- a/parsers/jscript.c +++ b/parsers/jscript.c @@ -311,7 +311,8 @@ static const keywordTable JsKeywordTable [] = { /* Recursive functions */ static void readTokenFull (tokenInfo *const token, bool include_newlines, vString *const repr); -static void skipArgumentList (tokenInfo *const token, bool include_newlines, vString *const repr); +static void skipArgumentList (tokenInfo *const token, bool include_newlines); +static void skipParameterList (tokenInfo *const token, bool include_newlines, vString *const repr); static bool parseFunction (tokenInfo *const token, tokenInfo *const name, const bool is_inside_class); static bool parseBlock (tokenInfo *const token, int parent_scope); static bool parseMethods (tokenInfo *const token, int class_index, const bool is_es6_class); @@ -1359,7 +1360,10 @@ static void skipBabelDecorator (tokenInfo *token, bool include_newlines, vString if (isType (token, TOKEN_OPEN_PAREN)) { /* @(complex ? dec1 : dec2) */ - skipArgumentList (token, include_newlines, repr); + if (repr) + skipParameterList (token, include_newlines, repr); + else + skipArgumentList (token, include_newlines); TRACE_PRINT ("found @(...) style decorator"); } else if (isType (token, TOKEN_IDENTIFIER)) @@ -1382,7 +1386,10 @@ static void skipBabelDecorator (tokenInfo *token, bool include_newlines, vString found_period = true; else if (isType (token, TOKEN_OPEN_PAREN)) { - skipArgumentList (token, include_newlines, repr); + if (repr) + skipParameterList (token, include_newlines, repr); + else + skipArgumentList (token, include_newlines); TRACE_PRINT("found @foo(...) style decorator"); break; } @@ -1459,20 +1466,17 @@ static int parseMethodsInAnonymousObject (tokenInfo *const token) return index; } -static void skipArgumentList (tokenInfo *const token, bool include_newlines, vString *const repr) +static void skipArgumentList (tokenInfo *const token, bool include_newlines) { TRACE_ENTER(); if (isType (token, TOKEN_OPEN_PAREN)) /* arguments? */ { int nest_level = 1; - if (repr) - vStringPut (repr, '('); - tokenType prev_token_type = token->type; while (nest_level > 0 && ! isType (token, TOKEN_EOF)) { - readTokenFull (token, false, repr); + readToken (token); if (isType (token, TOKEN_OPEN_PAREN)) nest_level++; else if (isType (token, TOKEN_CLOSE_PAREN)) @@ -1494,6 +1498,30 @@ static void skipArgumentList (tokenInfo *const token, bool include_newlines, vSt TRACE_LEAVE(); } +static void skipParameterList (tokenInfo *const token, bool include_newlines, vString *const repr) +{ + TRACE_ENTER_TEXT("repr = %p", repr); + + Assert (repr); + if (isType (token, TOKEN_OPEN_PAREN)) /* parameter? */ + { + int nest_level = 1; + if (repr) + vStringPut (repr, '('); + + while (nest_level > 0 && ! isType (token, TOKEN_EOF)) + { + readTokenFull (token, false, repr); + if (isType (token, TOKEN_OPEN_PAREN)) + nest_level++; + else if (isType (token, TOKEN_CLOSE_PAREN)) + nest_level--; + } + readTokenFull (token, include_newlines, NULL); + } + TRACE_LEAVE(); +} + static void skipArrayList (tokenInfo *const token, bool include_newlines) { /* @@ -1567,7 +1595,7 @@ static bool findCmdTerm (tokenInfo *const token, bool include_newlines, bool inc readTokenFull (token, include_newlines, NULL); } else if ( isType (token, TOKEN_OPEN_PAREN) ) - skipArgumentList(token, include_newlines, NULL); + skipArgumentList(token, include_newlines); else if ( isType (token, TOKEN_OPEN_SQUARE) ) skipArrayList(token, include_newlines); else @@ -1595,7 +1623,7 @@ static void parseSwitch (tokenInfo *const token) if (isType (token, TOKEN_OPEN_PAREN)) { - skipArgumentList(token, false, NULL); + skipArgumentList(token, false); } if (isType (token, TOKEN_OPEN_CURLY)) @@ -1634,7 +1662,7 @@ static bool parseLoop (tokenInfo *const token) readToken(token); if (isType (token, TOKEN_OPEN_PAREN)) - skipArgumentList(token, false, NULL); + skipArgumentList(token, false); if (isType (token, TOKEN_OPEN_CURLY)) parseBlock (token, CORK_NIL); @@ -1658,7 +1686,7 @@ static bool parseLoop (tokenInfo *const token) readToken(token); if (isType (token, TOKEN_OPEN_PAREN)) - skipArgumentList(token, true, NULL); + skipArgumentList(token, true); if (! isType (token, TOKEN_SEMICOLON)) { @@ -1728,7 +1756,7 @@ static bool parseIf (tokenInfo *const token) } if (isType (token, TOKEN_OPEN_PAREN)) - skipArgumentList(token, false, NULL); + skipArgumentList(token, false); if (isType (token, TOKEN_OPEN_CURLY)) parseBlock (token, CORK_NIL); @@ -1854,7 +1882,7 @@ static bool parseFunction (tokenInfo *const token, tokenInfo *const lhs_name, co readToken (token); if ( isType (token, TOKEN_OPEN_PAREN) ) - skipArgumentList(token, false, signature); + skipParameterList(token, false, signature); if ( isType (token, TOKEN_OPEN_CURLY) ) { @@ -2194,7 +2222,7 @@ static bool parseMethods (tokenInfo *const token, int class_index, } if ( isType (token, TOKEN_OPEN_PAREN) ) { - skipArgumentList(token, false, signature); + skipParameterList(token, false, signature); } function: @@ -2243,7 +2271,7 @@ static bool parseMethods (tokenInfo *const token, int class_index, else if (isType (token, TOKEN_OPEN_PAREN)) { vStringClear (signature); - skipArgumentList (token, false, signature); + skipParameterList (token, false, signature); } else if (isType (token, TOKEN_OPEN_SQUARE)) { @@ -2514,8 +2542,12 @@ static bool parsePrototype (tokenInfo *const name, tokenInfo *const token, state ! isType (method_body_token, TOKEN_EOF)) { if ( isType (method_body_token, TOKEN_OPEN_PAREN) ) - skipArgumentList(method_body_token, false, - vStringLength (signature) == 0 ? signature : NULL); + { + if (vStringIsEmpty (signature)) + skipParameterList(method_body_token, false, signature); + else + skipArgumentList(method_body_token, false); + } else { char* s1 = vStringValue (identifier_token->string); @@ -2762,7 +2794,7 @@ static bool parseStatementRHS (tokenInfo *const name, tokenInfo *const token, st readToken (token); if ( isType (token, TOKEN_OPEN_PAREN) ) - skipArgumentList(token, true, NULL); + skipArgumentList(token, true); if (isType (token, TOKEN_SEMICOLON) && token->nestLevel == 0) { @@ -3158,7 +3190,7 @@ static bool parseStatement (tokenInfo *const token, bool is_inside_class) readTokenFull (token, true, NULL); if ( isType (token, TOKEN_OPEN_PAREN) ) - skipArgumentList(token, false, NULL); + skipArgumentList(token, false); if ( isType (token, TOKEN_OPEN_SQUARE) ) skipArrayList(token, false); From 63771301a3eaf2a17a38c08063b6a21352fe34d6 Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Fri, 13 Jan 2023 08:08:24 +0900 Subject: [PATCH 4/7] JavaScript: make trace output easier to read by adding quote characters --- parsers/jscript.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parsers/jscript.c b/parsers/jscript.c index e937d280da..25aa47f837 100644 --- a/parsers/jscript.c +++ b/parsers/jscript.c @@ -524,7 +524,7 @@ static int makeJsTagCommon (const tokenInfo *const token, const jsKind kind, { const char *scope_str = getNameStringForCorkIndex (scope); const char *scope_kind_str = getKindStringForCorkIndex (scope); - TRACE_PRINT("Emitting tag for symbol '%s' of kind %s with scope '%s:%s'", name, kindName(kind), scope_kind_str, scope_str); + TRACE_PRINT("Emitting tag for symbol '%s' of kind '%s' with scope '%s:%s'", name, kindName(kind), scope_kind_str, scope_str); } #endif @@ -3446,13 +3446,13 @@ static void dumpToken (const tokenInfo *const token) if (strcmp(scope_str, "placeholder") == 0) { - TRACE_PRINT("%s: %s", + TRACE_PRINT("%s: '%s'", tokenTypeName (token->type), vStringValue (token->string)); } else { - TRACE_PRINT("%s: %s (scope '%s' of kind %s)", + TRACE_PRINT("%s: '%s' (scope '%s' of kind '%s')", tokenTypeName (token->type), vStringValue (token->string), scope_str, scope_kind_str); From b278dd3277947d525827ed0a245f7b604275f743 Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Fri, 13 Jan 2023 08:32:58 +0900 Subject: [PATCH 5/7] JavaScript: extract name in { ..., name, ...}, a shorthand for { ..., name: name, ...} Signed-off-by: Masatake YAMATO --- .../js-destructural-binding.d/expected.tags | 5 ++ .../js-destructural-binding.d/input-2.js | 8 +++ parsers/jscript.c | 54 +++++++++++++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/input-2.js diff --git a/Units/parser-javascript.r/js-destructural-binding.d/expected.tags b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags index aeb61943da..a9294b92f3 100644 --- a/Units/parser-javascript.r/js-destructural-binding.d/expected.tags +++ b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags @@ -64,3 +64,8 @@ f input-1.js /^function f({ u, x }) {$/;" f signature:({ u, x }) anonymousObjectf91cef720105 input-1.js /^f({u: 1, x: 2})$/;" v u input-1.js /^f({u: 1, x: 2})$/;" p variable:anonymousObjectf91cef720105 x input-1.js /^f({u: 1, x: 2})$/;" p variable:anonymousObjectf91cef720105 +f input-2.js /^function f(x, y, z) {$/;" f signature:(x, y, z) +anonymousObjectf91d7bd30105 input-2.js /^f({x, y, z})$/;" v +x input-2.js /^f({x, y, z})$/;" p variable:anonymousObjectf91d7bd30105 +y input-2.js /^f({x, y, z})$/;" p variable:anonymousObjectf91d7bd30105 +z input-2.js /^f({x, y, z})$/;" p variable:anonymousObjectf91d7bd30105 diff --git a/Units/parser-javascript.r/js-destructural-binding.d/input-2.js b/Units/parser-javascript.r/js-destructural-binding.d/input-2.js new file mode 100644 index 0000000000..d024f3bbca --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/input-2.js @@ -0,0 +1,8 @@ +function f(x, y, z) { + return x + y + z +} + +x = 1 +y = 1 +z = 1 +f({x, y, z}) diff --git a/parsers/jscript.c b/parsers/jscript.c index 25aa47f837..1f67125fe5 100644 --- a/parsers/jscript.c +++ b/parsers/jscript.c @@ -2196,13 +2196,45 @@ static bool parseMethods (tokenInfo *const token, int class_index, is_shorthand = isType (token, TOKEN_OPEN_PAREN); bool can_be_field = isType (token, TOKEN_EQUAL_SIGN); - if ( isType (token, TOKEN_COLON) || can_be_field || is_shorthand ) + /* is_comma is for handling + * + * { ..., name, ... } + * + * . This is shorthand of + * + * { ... ,name: name, ... } + * + * . + * In this case, the token variables point to: + * + * { ..., name, ... } + * name-------^ ^ + * token----------+ + * + */ + bool is_comma = ((!is_es6_class && isType (token, TOKEN_CLOSE_CURLY)) || isType (token, TOKEN_COMMA)); + if ( isType (token, TOKEN_COLON) || is_comma || can_be_field || is_shorthand ) { + tokenInfo * comma = NULL; if (! is_shorthand) { - readToken (token); - if (isKeyword (token, KEYWORD_async)) + if (is_comma) + { + comma = newToken (); + copyToken (comma, token, true); + copyToken (token, name, true); + /* + * { ..., name, ... } + * token -----^ ^ + * comma ---------+ + */ + } + else + { readToken (token); + if (isKeyword (token, KEYWORD_async)) + readToken (token); + } } vString * signature = vStringNew (); @@ -2294,7 +2326,19 @@ static bool parseMethods (tokenInfo *const token, int class_index, else { copyToken (saved_token, token, true); - readToken (token); + if (comma) + { + copyToken(token, comma, true); + deleteToken (comma); + comma = NULL; + /* + * { ..., name, ... } + * ^ + * token ---------+ + */ + } + else + readToken (token); } } deleteToken (saved_token); @@ -2316,6 +2360,8 @@ static bool parseMethods (tokenInfo *const token, int class_index, } vStringDelete (signature); + if (comma) + deleteToken (comma); } else { From 1761ee15d20f33e5ba116e53f9a71238a34eb7aa Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Tue, 24 Dec 2024 11:41:50 +0900 Subject: [PATCH 6/7] JavaScript: recognize "Computed object property names and destructuring" A computed object property, surrounded by [ and ], should not be extracted as variable. Signed-off-by: Masatake YAMATO --- .../js-destructural-binding.d/expected.tags | 9 ++++++ .../js-destructural-binding.d/input-3.js | 7 +++++ parsers/jscript.c | 29 ++++++++++++++++--- 3 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 Units/parser-javascript.r/js-destructural-binding.d/input-3.js diff --git a/Units/parser-javascript.r/js-destructural-binding.d/expected.tags b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags index a9294b92f3..303ab48e54 100644 --- a/Units/parser-javascript.r/js-destructural-binding.d/expected.tags +++ b/Units/parser-javascript.r/js-destructural-binding.d/expected.tags @@ -69,3 +69,12 @@ anonymousObjectf91d7bd30105 input-2.js /^f({x, y, z})$/;" v x input-2.js /^f({x, y, z})$/;" p variable:anonymousObjectf91d7bd30105 y input-2.js /^f({x, y, z})$/;" p variable:anonymousObjectf91d7bd30105 z input-2.js /^f({x, y, z})$/;" p variable:anonymousObjectf91d7bd30105 +key0 input-3.js /^let key0 = 'z';$/;" v +foo input-3.js /^let {[key0]: foo} = {z: 'alpha'};$/;" v +z input-3.js /^let {[key0]: foo} = {z: 'alpha'};$/;" p variable:anonymousObjectf91e08340105 +anonymousObjectf91e08340105 input-3.js /^let {[key0]: foo} = {z: 'alpha'};$/;" v +key1 input-3.js /^let key1 = 'x';$/;" v +bar input-3.js /^let {[key0]: bar = 'X', [key1]: baz} = {x: 'beta'};$/;" v +baz input-3.js /^let {[key0]: bar = 'X', [key1]: baz} = {x: 'beta'};$/;" v +x input-3.js /^let {[key0]: bar = 'X', [key1]: baz} = {x: 'beta'};$/;" p variable:anonymousObjectf91e08340205 +anonymousObjectf91e08340205 input-3.js /^let {[key0]: bar = 'X', [key1]: baz} = {x: 'beta'};$/;" v diff --git a/Units/parser-javascript.r/js-destructural-binding.d/input-3.js b/Units/parser-javascript.r/js-destructural-binding.d/input-3.js new file mode 100644 index 0000000000..4f25ed578c --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding.d/input-3.js @@ -0,0 +1,7 @@ +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Destructuring_assignment +// - Computed object property names and destructuring +let key0 = 'z'; +let {[key0]: foo} = {z: 'alpha'}; + +let key1 = 'x'; +let {[key0]: bar = 'X', [key1]: baz} = {x: 'beta'}; diff --git a/parsers/jscript.c b/parsers/jscript.c index 1f67125fe5..e8a58fd4f8 100644 --- a/parsers/jscript.c +++ b/parsers/jscript.c @@ -3044,10 +3044,31 @@ static bool parseObjectDestructuring (tokenInfo *const token, bool is_const) } else if (isType (token, TOKEN_OPEN_SQUARE)) { - if (parseArrayDestructuring (token, is_const)) - found = true; - if (state == OBJ_DESTRUCTURING_COLON) - state = OBJ_DESTRUCTURING_TAGGED; + if (state == OBJ_DESTRUCTURING_START) + { + /* + * @ > @ > + * let { [k0]: v0, [k1]: v1 = 0, v3 }; + * ^.....|...^.....|.......^.....: start + * ^.........^.............: colon + * + * We are at '@' in this context. + * Let's skip to '>'. + */ + skipArrayList(token, true); + if (isType (token, TOKEN_COLON)) + { + vStringClear (name->string); + state = OBJ_DESTRUCTURING_COLON; + } + } + else + { + if (parseArrayDestructuring (token, is_const)) + found = true; + if (state == OBJ_DESTRUCTURING_COLON) + state = OBJ_DESTRUCTURING_TAGGED; + } } else if (isType (token, TOKEN_IDENTIFIER)) { From b719aec1c58125a03cad49799165927f396d56a7 Mon Sep 17 00:00:00 2001 From: Masatake YAMATO Date: Tue, 31 Dec 2024 07:13:50 +0900 Subject: [PATCH 7/7] JavaScript,Units: add a case describing TODO items in object destructing (#3435) Signed-off-by: Masatake YAMATO --- .../js-destructural-binding-todo.b/args.ctags | 1 + .../js-destructural-binding-todo.b/expected.tags | 0 .../js-destructural-binding-todo.b/input-1.js | 3 +++ .../js-destructural-binding-todo.b/input.js | 9 +++++++++ 4 files changed, 13 insertions(+) create mode 100644 Units/parser-javascript.r/js-destructural-binding-todo.b/args.ctags create mode 100644 Units/parser-javascript.r/js-destructural-binding-todo.b/expected.tags create mode 100644 Units/parser-javascript.r/js-destructural-binding-todo.b/input-1.js create mode 100644 Units/parser-javascript.r/js-destructural-binding-todo.b/input.js diff --git a/Units/parser-javascript.r/js-destructural-binding-todo.b/args.ctags b/Units/parser-javascript.r/js-destructural-binding-todo.b/args.ctags new file mode 100644 index 0000000000..5ee5f79f70 --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding-todo.b/args.ctags @@ -0,0 +1 @@ +--sort=no diff --git a/Units/parser-javascript.r/js-destructural-binding-todo.b/expected.tags b/Units/parser-javascript.r/js-destructural-binding-todo.b/expected.tags new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Units/parser-javascript.r/js-destructural-binding-todo.b/input-1.js b/Units/parser-javascript.r/js-destructural-binding-todo.b/input-1.js new file mode 100644 index 0000000000..5f2714efce --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding-todo.b/input-1.js @@ -0,0 +1,3 @@ +// See #3435. +// f should be tagged with function kind. +let [f] = [function() {}]; diff --git a/Units/parser-javascript.r/js-destructural-binding-todo.b/input.js b/Units/parser-javascript.r/js-destructural-binding-todo.b/input.js new file mode 100644 index 0000000000..51b38a2ef5 --- /dev/null +++ b/Units/parser-javascript.r/js-destructural-binding-todo.b/input.js @@ -0,0 +1,9 @@ +// See #3435. +// If an object literal is specified as a default value in object restructuring, +// the parser may fail to extract the variable (or constant): +var{ c = {a: 1} } = { c: undefined }; +var{ d = {a: 1} } = {d: 3}; +var a = 1 +var [x = {a: 2}, y] = [, 4]; +var [x = [a, 2], z] = [, 4]; +var { 'alpha': q = {'x': 9} } = {'alpha': 3};