From bd5c2a20d27df3786b8c748f06cadf0658ab2e65 Mon Sep 17 00:00:00 2001 From: Shahar Soel <4233843+bd82@users.noreply.github.com> Date: Sun, 16 Jan 2022 22:10:34 +0200 Subject: [PATCH] feat: expose cts-dts-gen on root chevrotain package (#1713) --- .prettierignore | 1 + examples/implementation_languages/README.md | 3 + .../implementation_languages/package.json | 2 +- .../typescript/json_cst.d.ts | 69 ++++ .../typescript/scripts/gen_dts_signatures.js | 12 + .../typescript/typescript_json.ts | 4 +- packages/chevrotain/package.json | 2 + packages/chevrotain/src/api.ts | 9 +- .../chevrotain/src/parse/errors_public.ts | 9 +- .../chevrotain/src/parse/grammar/checks.ts | 6 +- .../chevrotain/src/parse/grammar/first.ts | 20 +- .../chevrotain/src/parse/grammar/follow.ts | 2 +- .../chevrotain/src/parse/grammar/gast/gast.ts | 188 ----------- .../grammar/gast/gast_resolver_public.ts | 5 +- .../src/parse/grammar/interpreter.ts | 8 +- .../chevrotain/src/parse/grammar/lookahead.ts | 9 +- .../chevrotain/src/parse/grammar/resolver.ts | 4 +- packages/chevrotain/src/parse/grammar/rest.ts | 5 +- .../chevrotain/src/parse/parser/parser.ts | 2 +- .../src/parse/parser/traits/gast_recorder.ts | 2 +- .../src/parse/parser/traits/looksahead.ts | 85 ++++- .../src/parse/parser/traits/recognizer_api.ts | 2 +- .../parse/parser/traits/recognizer_engine.ts | 2 +- packages/chevrotain/src/scan/lexer_public.ts | 2 +- .../test/parse/grammar/checks_spec.ts | 2 +- .../test/parse/grammar/first_spec.ts | 7 +- .../test/parse/grammar/follow_spec.ts | 2 +- .../test/parse/grammar/interperter_spec.ts | 2 +- .../test/parse/grammar/lookahead_spec.ts | 2 +- .../test/parse/grammar/resolver_spec.ts | 2 +- packages/cst-dts-gen/api.d.ts | 12 - packages/cst-dts-gen/package.json | 9 +- packages/cst-dts-gen/src/api.ts | 13 +- packages/cst-dts-gen/src/generate.ts | 28 +- packages/cst-dts-gen/src/model.ts | 7 +- packages/cst-dts-gen/test/options_spec.ts | 27 +- packages/cst-dts-gen/test/sample_test.ts | 13 +- packages/gast/.mocharc.js | 6 + packages/gast/nyc.config.js | 1 + packages/gast/package.json | 42 +++ packages/gast/src/api.ts | 23 ++ packages/gast/src/helpers.ts | 101 ++++++ .../gast/gast_public.ts => gast/src/model.ts} | 31 +- .../src/visitor.ts} | 12 +- packages/gast/test/helpers_spec.ts | 308 ++++++++++++++++++ .../gast_spec.ts => gast/test/model_spec.ts} | 30 +- packages/gast/test/visitor_spec.ts | 249 ++++++++++++++ packages/gast/tsconfig.json | 9 + packages/types/api.d.ts | 33 ++ .../docs/guide/concrete_syntax_tree.md | 70 +++- tsconfig.json | 5 +- 51 files changed, 1146 insertions(+), 353 deletions(-) create mode 100644 examples/implementation_languages/typescript/json_cst.d.ts create mode 100644 examples/implementation_languages/typescript/scripts/gen_dts_signatures.js delete mode 100644 packages/chevrotain/src/parse/grammar/gast/gast.ts delete mode 100644 packages/cst-dts-gen/api.d.ts create mode 100644 packages/gast/.mocharc.js create mode 100644 packages/gast/nyc.config.js create mode 100644 packages/gast/package.json create mode 100644 packages/gast/src/api.ts create mode 100644 packages/gast/src/helpers.ts rename packages/{chevrotain/src/parse/grammar/gast/gast_public.ts => gast/src/model.ts} (93%) rename packages/{chevrotain/src/parse/grammar/gast/gast_visitor_public.ts => gast/src/visitor.ts} (67%) create mode 100644 packages/gast/test/helpers_spec.ts rename packages/{chevrotain/test/parse/grammar/gast_spec.ts => gast/test/model_spec.ts} (95%) create mode 100644 packages/gast/test/visitor_spec.ts create mode 100644 packages/gast/tsconfig.json diff --git a/.prettierignore b/.prettierignore index 806256853..c4460e6f4 100644 --- a/.prettierignore +++ b/.prettierignore @@ -12,6 +12,7 @@ /examples/webpack/lib /examples/implementation_languages/typescript/*.js +/examples/implementation_languages/typescript/*.d.ts /examples/implementation_languages/coffeescript/*.js /examples/parser/minification/gen/ diff --git a/examples/implementation_languages/README.md b/examples/implementation_languages/README.md index ea8ebea0a..96c2c9195 100644 --- a/examples/implementation_languages/README.md +++ b/examples/implementation_languages/README.md @@ -12,6 +12,9 @@ The following examples all implement the same JSON parser using a variety of imp - [TypeScript](https://github.com/chevrotain/chevrotain/blob/master/examples/implementation_languages/typescript/typescript_json.ts) + - [Generated CST d.ts signatures](https://github.com/chevrotain/chevrotain/blob/master/examples/implementation_languages/typescript/json_cst.d.ts) + - [Script to generate CST d.ts signatures](https://github.com/chevrotain/chevrotain/blob/master/examples/implementation_languages/typescript/scripts/gen_dts_signatures.js) + - [CoffeeScript](https://github.com/chevrotain/chevrotain/blob/master/examples/implementation_languages/coffeescript/coffeescript_json.coffee) To run all the implementation languages examples tests: diff --git a/examples/implementation_languages/package.json b/examples/implementation_languages/package.json index d37732519..19f4d836c 100644 --- a/examples/implementation_languages/package.json +++ b/examples/implementation_languages/package.json @@ -3,7 +3,7 @@ "version": "9.1.0", "scripts": { "build": "npm-run-all build:ts build:coffee", - "build:ts": "tsc ./typescript/typescript_json.ts --types \" \"", + "build:ts": "tsc ./typescript/typescript_json.ts --types \" \" && node ./typescript/scripts/gen_dts_signatures.js", "build:coffee": "coffee -c ./coffeescript/coffeescript_json.coffee", "test": "npm-run-all test:*", "test:cjs": "mocha \"*spec.js\"", diff --git a/examples/implementation_languages/typescript/json_cst.d.ts b/examples/implementation_languages/typescript/json_cst.d.ts new file mode 100644 index 000000000..d9cc39b52 --- /dev/null +++ b/examples/implementation_languages/typescript/json_cst.d.ts @@ -0,0 +1,69 @@ +import type { CstNode, ICstVisitor, IToken } from "chevrotain"; + +export interface JsonCstNode extends CstNode { + name: "json"; + children: JsonCstChildren; +} + +export type JsonCstChildren = { + object?: ObjectCstNode[]; + array?: ArrayCstNode[]; +}; + +export interface ObjectCstNode extends CstNode { + name: "object"; + children: ObjectCstChildren; +} + +export type ObjectCstChildren = { + LCurly: IToken[]; + objectItem?: ObjectItemCstNode[]; + Comma?: IToken[]; + RCurly: IToken[]; +}; + +export interface ObjectItemCstNode extends CstNode { + name: "objectItem"; + children: ObjectItemCstChildren; +} + +export type ObjectItemCstChildren = { + StringLiteral: IToken[]; + Colon: IToken[]; + value: ValueCstNode[]; +}; + +export interface ArrayCstNode extends CstNode { + name: "array"; + children: ArrayCstChildren; +} + +export type ArrayCstChildren = { + LSquare: IToken[]; + value?: ValueCstNode[]; + Comma?: IToken[]; + RSquare: IToken[]; +}; + +export interface ValueCstNode extends CstNode { + name: "value"; + children: ValueCstChildren; +} + +export type ValueCstChildren = { + StringLiteral?: IToken[]; + NumberLiteral?: IToken[]; + object?: ObjectCstNode[]; + array?: ArrayCstNode[]; + True?: IToken[]; + False?: IToken[]; + Null?: IToken[]; +}; + +export interface ICstNodeVisitor extends ICstVisitor { + json(children: JsonCstChildren, param?: IN): OUT; + object(children: ObjectCstChildren, param?: IN): OUT; + objectItem(children: ObjectItemCstChildren, param?: IN): OUT; + array(children: ArrayCstChildren, param?: IN): OUT; + value(children: ValueCstChildren, param?: IN): OUT; +} diff --git a/examples/implementation_languages/typescript/scripts/gen_dts_signatures.js b/examples/implementation_languages/typescript/scripts/gen_dts_signatures.js new file mode 100644 index 000000000..5b95eabc2 --- /dev/null +++ b/examples/implementation_languages/typescript/scripts/gen_dts_signatures.js @@ -0,0 +1,12 @@ +/** + * This is a minimal script that generates TypeScript definitions + * from a Chevrotain parser. + */ +const { writeFileSync } = require("fs") +const { resolve } = require("path") +const { generateCstDts } = require("chevrotain") +const { productions } = require("../typescript_json") + +const dtsString = generateCstDts(productions) +const dtsPath = resolve(__dirname, "..", "json_cst.d.ts") +writeFileSync(dtsPath, dtsString) diff --git a/examples/implementation_languages/typescript/typescript_json.ts b/examples/implementation_languages/typescript/typescript_json.ts index 2847d95f8..770e254af 100644 --- a/examples/implementation_languages/typescript/typescript_json.ts +++ b/examples/implementation_languages/typescript/typescript_json.ts @@ -1,4 +1,4 @@ -import { createToken, Lexer, CstParser } from "chevrotain" +import { createToken, Lexer, CstParser, Rule } from "chevrotain" const True = createToken({ name: "True", pattern: /true/ }) const False = createToken({ name: "False", pattern: /false/ }) @@ -102,6 +102,8 @@ class JsonParserTypeScript extends CstParser { // reuse the same parser instance. const parser = new JsonParserTypeScript() +export const productions: Record = parser.getGAstProductions() + export function parseJson(text) { const lexResult = JsonLexer.tokenize(text) // setting a new input will RESET the parser instance's state. diff --git a/packages/chevrotain/package.json b/packages/chevrotain/package.json index 07b3896e3..c82722d59 100644 --- a/packages/chevrotain/package.json +++ b/packages/chevrotain/package.json @@ -72,6 +72,8 @@ }, "dependencies": { "@chevrotain/types": "^9.1.0", + "@chevrotain/gast": "^9.1.0", + "@chevrotain/cst-dts-gen": "^9.1.0", "@chevrotain/utils": "^9.1.0", "regexp-to-ast": "0.5.0", "lodash": "4.17.21" diff --git a/packages/chevrotain/src/api.ts b/packages/chevrotain/src/api.ts index 81e6a37d3..f7506c13e 100644 --- a/packages/chevrotain/src/api.ts +++ b/packages/chevrotain/src/api.ts @@ -48,16 +48,17 @@ export { RepetitionWithSeparator, Rule, Terminal -} from "./parse/grammar/gast/gast_public" +} from "@chevrotain/gast" // GAST Utilities export { serializeGrammar, - serializeProduction -} from "./parse/grammar/gast/gast_public" + serializeProduction, + GAstVisitor +} from "@chevrotain/gast" -export { GAstVisitor } from "./parse/grammar/gast/gast_visitor_public" +export { generateCstDts } from "@chevrotain/cst-dts-gen" /* istanbul ignore next */ export function clearCache() { diff --git a/packages/chevrotain/src/parse/errors_public.ts b/packages/chevrotain/src/parse/errors_public.ts index 6926a9b96..1c0fa0bc3 100644 --- a/packages/chevrotain/src/parse/errors_public.ts +++ b/packages/chevrotain/src/parse/errors_public.ts @@ -2,13 +2,8 @@ import { hasTokenLabel, tokenLabel } from "../scan/tokens_public" import first from "lodash/first" import map from "lodash/map" import reduce from "lodash/reduce" -import { - Alternation, - NonTerminal, - Rule, - Terminal -} from "./grammar/gast/gast_public" -import { getProductionDslName } from "./grammar/gast/gast" +import { Alternation, NonTerminal, Rule, Terminal } from "@chevrotain/gast" +import { getProductionDslName } from "@chevrotain/gast" import { IParserErrorMessageProvider, IProductionWithOccurrence, diff --git a/packages/chevrotain/src/parse/grammar/checks.ts b/packages/chevrotain/src/parse/grammar/checks.ts index e6db8bc60..fa6b0964d 100644 --- a/packages/chevrotain/src/parse/grammar/checks.ts +++ b/packages/chevrotain/src/parse/grammar/checks.ts @@ -20,7 +20,7 @@ import { IParserEmptyAlternativeDefinitionError, ParserDefinitionErrorType } from "../parser/parser" -import { getProductionDslName, isOptionalProd } from "./gast/gast" +import { getProductionDslName, isOptionalProd } from "@chevrotain/gast" import { Alternative, containsPath, @@ -41,8 +41,8 @@ import { RepetitionWithSeparator, Rule, Terminal -} from "./gast/gast_public" -import { GAstVisitor } from "./gast/gast_visitor_public" +} from "@chevrotain/gast" +import { GAstVisitor } from "@chevrotain/gast" import { IProduction, IProductionWithOccurrence, diff --git a/packages/chevrotain/src/parse/grammar/first.ts b/packages/chevrotain/src/parse/grammar/first.ts index 6b4c0fc02..ac66c06cd 100644 --- a/packages/chevrotain/src/parse/grammar/first.ts +++ b/packages/chevrotain/src/parse/grammar/first.ts @@ -1,8 +1,12 @@ import flatten from "lodash/flatten" import uniq from "lodash/uniq" import map from "lodash/map" -import { AbstractProduction, NonTerminal, Terminal } from "./gast/gast_public" -import { isBranchingProd, isOptionalProd, isSequenceProd } from "./gast/gast" +import { NonTerminal, Terminal } from "@chevrotain/gast" +import { + isBranchingProd, + isOptionalProd, + isSequenceProd +} from "@chevrotain/gast" import { IProduction, TokenType } from "@chevrotain/types" export function first(prod: IProduction): TokenType[] { @@ -20,15 +24,17 @@ export function first(prod: IProduction): TokenType[] { } else if (prod instanceof Terminal) { return firstForTerminal(prod) } else if (isSequenceProd(prod)) { - return firstForSequence(prod) + return firstForSequence(prod) } else if (isBranchingProd(prod)) { - return firstForBranching(prod) + return firstForBranching(prod) } else { throw Error("non exhaustive match") } } -export function firstForSequence(prod: AbstractProduction): TokenType[] { +export function firstForSequence(prod: { + definition: IProduction[] +}): TokenType[] { let firstSet: TokenType[] = [] const seq = prod.definition let nextSubProdIdx = 0 @@ -48,7 +54,9 @@ export function firstForSequence(prod: AbstractProduction): TokenType[] { return uniq(firstSet) } -export function firstForBranching(prod: AbstractProduction): TokenType[] { +export function firstForBranching(prod: { + definition: IProduction[] +}): TokenType[] { const allAlternativesFirsts: TokenType[][] = map( prod.definition, (innerProd) => { diff --git a/packages/chevrotain/src/parse/grammar/follow.ts b/packages/chevrotain/src/parse/grammar/follow.ts index beb70ce8c..48374e7c5 100644 --- a/packages/chevrotain/src/parse/grammar/follow.ts +++ b/packages/chevrotain/src/parse/grammar/follow.ts @@ -3,7 +3,7 @@ import { first } from "./first" import forEach from "lodash/forEach" import assign from "lodash/assign" import { IN } from "../constants" -import { Alternative, NonTerminal, Rule, Terminal } from "./gast/gast_public" +import { Alternative, NonTerminal, Rule, Terminal } from "@chevrotain/gast" import { IProduction, TokenType } from "@chevrotain/types" // This ResyncFollowsWalker computes all of the follows required for RESYNC diff --git a/packages/chevrotain/src/parse/grammar/gast/gast.ts b/packages/chevrotain/src/parse/grammar/gast/gast.ts deleted file mode 100644 index 0e6a47688..000000000 --- a/packages/chevrotain/src/parse/grammar/gast/gast.ts +++ /dev/null @@ -1,188 +0,0 @@ -import some from "lodash/some" -import every from "lodash/every" -import has from "lodash/has" -import includes from "lodash/includes" -import { - AbstractProduction, - Alternation, - Alternative, - NonTerminal, - Option, - Repetition, - RepetitionMandatory, - RepetitionMandatoryWithSeparator, - RepetitionWithSeparator, - Rule, - Terminal -} from "./gast_public" -import { GAstVisitor } from "./gast_visitor_public" -import { IProduction, IProductionWithOccurrence } from "@chevrotain/types" - -export function isSequenceProd(prod: IProduction): boolean { - return ( - prod instanceof Alternative || - prod instanceof Option || - prod instanceof Repetition || - prod instanceof RepetitionMandatory || - prod instanceof RepetitionMandatoryWithSeparator || - prod instanceof RepetitionWithSeparator || - prod instanceof Terminal || - prod instanceof Rule - ) -} - -export function isOptionalProd( - prod: IProduction, - alreadyVisited: NonTerminal[] = [] -): boolean { - const isDirectlyOptional = - prod instanceof Option || - prod instanceof Repetition || - prod instanceof RepetitionWithSeparator - if (isDirectlyOptional) { - return true - } - - // note that this can cause infinite loop if one optional empty TOP production has a cyclic dependency with another - // empty optional top rule - // may be indirectly optional ((A?B?C?) | (D?E?F?)) - if (prod instanceof Alternation) { - // for OR its enough for just one of the alternatives to be optional - return some((prod).definition, (subProd: IProduction) => { - return isOptionalProd(subProd, alreadyVisited) - }) - } else if (prod instanceof NonTerminal && includes(alreadyVisited, prod)) { - // avoiding stack overflow due to infinite recursion - return false - } else if (prod instanceof AbstractProduction) { - if (prod instanceof NonTerminal) { - alreadyVisited.push(prod) - } - return every( - (prod).definition, - (subProd: IProduction) => { - return isOptionalProd(subProd, alreadyVisited) - } - ) - } else { - return false - } -} - -export function isBranchingProd(prod: IProduction): boolean { - return prod instanceof Alternation -} - -export function getProductionDslName(prod: IProductionWithOccurrence): string { - /* istanbul ignore else */ - if (prod instanceof NonTerminal) { - return "SUBRULE" - } else if (prod instanceof Option) { - return "OPTION" - } else if (prod instanceof Alternation) { - return "OR" - } else if (prod instanceof RepetitionMandatory) { - return "AT_LEAST_ONE" - } else if (prod instanceof RepetitionMandatoryWithSeparator) { - return "AT_LEAST_ONE_SEP" - } else if (prod instanceof RepetitionWithSeparator) { - return "MANY_SEP" - } else if (prod instanceof Repetition) { - return "MANY" - } else if (prod instanceof Terminal) { - return "CONSUME" - } else { - throw Error("non exhaustive match") - } -} - -export class DslMethodsCollectorVisitor extends GAstVisitor { - // A minus is never valid in an identifier name - public separator = "-" - public dslMethods: { - [subruleOrTerminalName: string]: IProductionWithOccurrence[] - option: Option[] - alternation: Alternation[] - repetition: Repetition[] - repetitionWithSeparator: RepetitionWithSeparator[] - repetitionMandatory: RepetitionMandatory[] - repetitionMandatoryWithSeparator: RepetitionMandatoryWithSeparator[] - } = { - option: [], - alternation: [], - repetition: [], - repetitionWithSeparator: [], - repetitionMandatory: [], - repetitionMandatoryWithSeparator: [] - } - - reset() { - this.dslMethods = { - option: [], - alternation: [], - repetition: [], - repetitionWithSeparator: [], - repetitionMandatory: [], - repetitionMandatoryWithSeparator: [] - } - } - - public visitTerminal(terminal: Terminal): void { - const key = terminal.terminalType.name + this.separator + "Terminal" - if (!has(this.dslMethods, key)) { - this.dslMethods[key] = [] - } - this.dslMethods[key].push(terminal) - } - - public visitNonTerminal(subrule: NonTerminal): void { - const key = subrule.nonTerminalName + this.separator + "Terminal" - if (!has(this.dslMethods, key)) { - this.dslMethods[key] = [] - } - this.dslMethods[key].push(subrule) - } - - public visitOption(option: Option): void { - this.dslMethods.option.push(option) - } - - public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void { - this.dslMethods.repetitionWithSeparator.push(manySep) - } - - public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void { - this.dslMethods.repetitionMandatory.push(atLeastOne) - } - - public visitRepetitionMandatoryWithSeparator( - atLeastOneSep: RepetitionMandatoryWithSeparator - ): void { - this.dslMethods.repetitionMandatoryWithSeparator.push(atLeastOneSep) - } - - public visitRepetition(many: Repetition): void { - this.dslMethods.repetition.push(many) - } - - public visitAlternation(or: Alternation): void { - this.dslMethods.alternation.push(or) - } -} - -const collectorVisitor = new DslMethodsCollectorVisitor() -export function collectMethods(rule: Rule): { - option: Option[] - alternation: Alternation[] - repetition: Repetition[] - repetitionWithSeparator: RepetitionWithSeparator[] - repetitionMandatory: RepetitionMandatory[] - repetitionMandatoryWithSeparator: RepetitionMandatoryWithSeparator[] -} { - collectorVisitor.reset() - rule.accept(collectorVisitor) - const dslMethods = collectorVisitor.dslMethods - // avoid uncleaned references - collectorVisitor.reset() - return dslMethods -} diff --git a/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts b/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts index d9f962d30..d92b8aaa7 100644 --- a/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts +++ b/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts @@ -1,4 +1,4 @@ -import { Rule } from "./gast_public" +import { Rule } from "@chevrotain/gast" import forEach from "lodash/forEach" import defaults from "lodash/defaults" import { resolveGrammar as orgResolveGrammar } from "../resolver" @@ -7,8 +7,7 @@ import { defaultGrammarResolverErrorProvider, defaultGrammarValidatorErrorProvider } from "../../errors_public" -import { DslMethodsCollectorVisitor } from "./gast" -import { IProductionWithOccurrence, TokenType } from "@chevrotain/types" +import { TokenType } from "@chevrotain/types" import { IGrammarResolverErrorMessageProvider, IGrammarValidatorErrorMessageProvider, diff --git a/packages/chevrotain/src/parse/grammar/interpreter.ts b/packages/chevrotain/src/parse/grammar/interpreter.ts index 56b56c096..b4c146562 100644 --- a/packages/chevrotain/src/parse/grammar/interpreter.ts +++ b/packages/chevrotain/src/parse/grammar/interpreter.ts @@ -9,7 +9,6 @@ import clone from "lodash/clone" import { first } from "./first" import { TokenMatcher } from "../parser/parser" import { - AbstractProduction, Alternation, Alternative, NonTerminal, @@ -20,7 +19,7 @@ import { RepetitionWithSeparator, Rule, Terminal -} from "./gast/gast_public" +} from "@chevrotain/gast" import { IGrammarPath, IProduction, @@ -65,7 +64,10 @@ export abstract class AbstractNextPossibleTokensWalker extends RestWalker { return this.possibleTokTypes } - walk(prod: AbstractProduction, prevRest: IProduction[] = []): void { + walk( + prod: { definition: IProduction[] }, + prevRest: IProduction[] = [] + ): void { // stop scanning once we found the path if (!this.found) { super.walk(prod, prevRest) diff --git a/packages/chevrotain/src/parse/grammar/lookahead.ts b/packages/chevrotain/src/parse/grammar/lookahead.ts index 5c179f27f..8e69585cc 100644 --- a/packages/chevrotain/src/parse/grammar/lookahead.ts +++ b/packages/chevrotain/src/parse/grammar/lookahead.ts @@ -13,7 +13,6 @@ import { tokenStructuredMatcherNoCategories } from "../../scan/tokens" import { - AbstractProduction, Alternation, Alternative as AlternativeGAST, Option, @@ -22,8 +21,8 @@ import { RepetitionMandatoryWithSeparator, RepetitionWithSeparator, Rule -} from "./gast/gast_public" -import { GAstVisitor } from "./gast/gast_visitor_public" +} from "@chevrotain/gast" +import { GAstVisitor } from "@chevrotain/gast" import { IOrAlt, IProduction, @@ -332,7 +331,7 @@ class RestDefinitionFinderWalker extends RestWalker { } private checkIsTarget( - node: AbstractProduction & IProductionWithOccurrence, + node: IProductionWithOccurrence, expectedProdType: PROD_TYPE, currRest: IProduction[], prevRest: IProduction[] @@ -437,7 +436,7 @@ class InsideDefinitionFinderVisitor extends GAstVisitor { } private checkIsTarget( - node: AbstractProduction & IProductionWithOccurrence, + node: { definition: IProduction[] } & IProductionWithOccurrence, expectedProdName: PROD_TYPE ): void { if ( diff --git a/packages/chevrotain/src/parse/grammar/resolver.ts b/packages/chevrotain/src/parse/grammar/resolver.ts index 46936e51c..0590bc36e 100644 --- a/packages/chevrotain/src/parse/grammar/resolver.ts +++ b/packages/chevrotain/src/parse/grammar/resolver.ts @@ -4,8 +4,8 @@ import { } from "../parser/parser" import forEach from "lodash/forEach" import values from "lodash/values" -import { NonTerminal, Rule } from "./gast/gast_public" -import { GAstVisitor } from "./gast/gast_visitor_public" +import { NonTerminal, Rule } from "@chevrotain/gast" +import { GAstVisitor } from "@chevrotain/gast" import { IGrammarResolverErrorMessageProvider, IParserDefinitionError diff --git a/packages/chevrotain/src/parse/grammar/rest.ts b/packages/chevrotain/src/parse/grammar/rest.ts index 468bf1a77..13a930883 100644 --- a/packages/chevrotain/src/parse/grammar/rest.ts +++ b/packages/chevrotain/src/parse/grammar/rest.ts @@ -1,7 +1,6 @@ import drop from "lodash/drop" import forEach from "lodash/forEach" import { - AbstractProduction, Alternation, Alternative, NonTerminal, @@ -11,14 +10,14 @@ import { RepetitionMandatoryWithSeparator, RepetitionWithSeparator, Terminal -} from "./gast/gast_public" +} from "@chevrotain/gast" import { IProduction } from "@chevrotain/types" /** * A Grammar Walker that computes the "remaining" grammar "after" a productions in the grammar. */ export abstract class RestWalker { - walk(prod: AbstractProduction, prevRest: any[] = []): void { + walk(prod: { definition: IProduction[] }, prevRest: any[] = []): void { forEach(prod.definition, (subProd: IProduction, index) => { const currRest = drop(prod.definition, index + 1) /* istanbul ignore else */ diff --git a/packages/chevrotain/src/parse/parser/parser.ts b/packages/chevrotain/src/parse/parser/parser.ts index 29a58cfd2..2e7b4188f 100644 --- a/packages/chevrotain/src/parse/parser/parser.ts +++ b/packages/chevrotain/src/parse/parser/parser.ts @@ -38,7 +38,7 @@ import { GastRecorder } from "./traits/gast_recorder" import { PerformanceTracer } from "./traits/perf_tracer" import { applyMixins } from "./utils/apply_mixins" import { IParserDefinitionError } from "../grammar/types" -import { Rule } from "../grammar/gast/gast_public" +import { Rule } from "@chevrotain/gast" import { IParserConfigInternal, ParserMethodInternal } from "./types" export const END_OF_FILE = createTokenInstance( diff --git a/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts b/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts index c05476f83..02702e50b 100644 --- a/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts +++ b/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts @@ -32,7 +32,7 @@ import { RepetitionWithSeparator, Rule, Terminal -} from "../../grammar/gast/gast_public" +} from "@chevrotain/gast" import { Lexer } from "../../../scan/lexer_public" import { augmentTokenTypes, hasShortKeyProperty } from "../../../scan/tokens" import { createToken, createTokenInstance } from "../../../scan/tokens_public" diff --git a/packages/chevrotain/src/parse/parser/traits/looksahead.ts b/packages/chevrotain/src/parse/parser/traits/looksahead.ts index 966730704..886393ba4 100644 --- a/packages/chevrotain/src/parse/parser/traits/looksahead.ts +++ b/packages/chevrotain/src/parse/parser/traits/looksahead.ts @@ -23,8 +23,17 @@ import { OR_IDX } from "../../grammar/keys" import { MixedInParser } from "./parser_traits" -import { Rule } from "../../grammar/gast/gast_public" -import { collectMethods, getProductionDslName } from "../../grammar/gast/gast" +import { + Alternation, + GAstVisitor, + Option, + Repetition, + RepetitionMandatory, + RepetitionMandatoryWithSeparator, + RepetitionWithSeparator, + Rule +} from "@chevrotain/gast" +import { getProductionDslName } from "@chevrotain/gast" /** * Trait responsible for the lookahead related utilities and optimizations. @@ -218,3 +227,75 @@ export class LooksAhead { this.lookAheadFuncsCache.set(key, value) } } + +class DslMethodsCollectorVisitor extends GAstVisitor { + public dslMethods: { + option: Option[] + alternation: Alternation[] + repetition: Repetition[] + repetitionWithSeparator: RepetitionWithSeparator[] + repetitionMandatory: RepetitionMandatory[] + repetitionMandatoryWithSeparator: RepetitionMandatoryWithSeparator[] + } = { + option: [], + alternation: [], + repetition: [], + repetitionWithSeparator: [], + repetitionMandatory: [], + repetitionMandatoryWithSeparator: [] + } + + reset() { + this.dslMethods = { + option: [], + alternation: [], + repetition: [], + repetitionWithSeparator: [], + repetitionMandatory: [], + repetitionMandatoryWithSeparator: [] + } + } + + public visitOption(option: Option): void { + this.dslMethods.option.push(option) + } + + public visitRepetitionWithSeparator(manySep: RepetitionWithSeparator): void { + this.dslMethods.repetitionWithSeparator.push(manySep) + } + + public visitRepetitionMandatory(atLeastOne: RepetitionMandatory): void { + this.dslMethods.repetitionMandatory.push(atLeastOne) + } + + public visitRepetitionMandatoryWithSeparator( + atLeastOneSep: RepetitionMandatoryWithSeparator + ): void { + this.dslMethods.repetitionMandatoryWithSeparator.push(atLeastOneSep) + } + + public visitRepetition(many: Repetition): void { + this.dslMethods.repetition.push(many) + } + + public visitAlternation(or: Alternation): void { + this.dslMethods.alternation.push(or) + } +} + +const collectorVisitor = new DslMethodsCollectorVisitor() +export function collectMethods(rule: Rule): { + option: Option[] + alternation: Alternation[] + repetition: Repetition[] + repetitionWithSeparator: RepetitionWithSeparator[] + repetitionMandatory: RepetitionMandatory[] + repetitionMandatoryWithSeparator: RepetitionMandatoryWithSeparator[] +} { + collectorVisitor.reset() + rule.accept(collectorVisitor) + const dslMethods = collectorVisitor.dslMethods + // avoid uncleaned references + collectorVisitor.reset() + return dslMethods +} diff --git a/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts b/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts index e1e2b6d64..a9ff4269b 100644 --- a/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts +++ b/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts @@ -20,7 +20,7 @@ import { DEFAULT_RULE_CONFIG, ParserDefinitionErrorType } from "../parser" import { defaultGrammarValidatorErrorProvider } from "../../errors_public" import { validateRuleIsOverridden } from "../../grammar/checks" import { MixedInParser } from "./parser_traits" -import { Rule, serializeGrammar } from "../../grammar/gast/gast_public" +import { Rule, serializeGrammar } from "@chevrotain/gast" import { IParserDefinitionError } from "../../grammar/types" import { ParserMethodInternal } from "../types" diff --git a/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts b/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts index 57401ccfa..e009f9d58 100644 --- a/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts +++ b/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts @@ -59,7 +59,7 @@ import { tokenStructuredMatcher, tokenStructuredMatcherNoCategories } from "../../../scan/tokens" -import { Rule } from "../../grammar/gast/gast_public" +import { Rule } from "@chevrotain/gast" import { ParserMethodInternal } from "../types" /** diff --git a/packages/chevrotain/src/scan/lexer_public.ts b/packages/chevrotain/src/scan/lexer_public.ts index a88939185..ec0d9b698 100644 --- a/packages/chevrotain/src/scan/lexer_public.ts +++ b/packages/chevrotain/src/scan/lexer_public.ts @@ -902,7 +902,7 @@ export class Lexer { } // Place holder, will be replaced by the correct variant according to the hasCustom flag option at runtime. - private handlePayload(token: IToken, payload: any): void {} + private handlePayload: (token: IToken, payload: any) => void private handlePayloadNoCustom(token: IToken, payload: any): void {} diff --git a/packages/chevrotain/test/parse/grammar/checks_spec.ts b/packages/chevrotain/test/parse/grammar/checks_spec.ts index 83560dee9..a191d22fd 100644 --- a/packages/chevrotain/test/parse/grammar/checks_spec.ts +++ b/packages/chevrotain/test/parse/grammar/checks_spec.ts @@ -28,7 +28,7 @@ import { RepetitionMandatory, Rule, Terminal -} from "../../../src/parse/grammar/gast/gast_public" +} from "@chevrotain/gast" import { defaultGrammarValidatorErrorProvider } from "../../../src/parse/errors_public" import { IToken, TokenType } from "@chevrotain/types" import { expect } from "chai" diff --git a/packages/chevrotain/test/parse/grammar/first_spec.ts b/packages/chevrotain/test/parse/grammar/first_spec.ts index b61884b25..0d2880f10 100644 --- a/packages/chevrotain/test/parse/grammar/first_spec.ts +++ b/packages/chevrotain/test/parse/grammar/first_spec.ts @@ -1,11 +1,6 @@ import { first } from "../../../src/parse/grammar/first" import { setEquality } from "../../utils/matchers" -import { - Alternative, - Terminal, - Option, - Alternation -} from "../../../src/parse/grammar/gast/gast_public" +import { Alternative, Terminal, Option, Alternation } from "@chevrotain/gast" import { expect } from "chai" import { createToken } from "../../../src/scan/tokens_public" diff --git a/packages/chevrotain/test/parse/grammar/follow_spec.ts b/packages/chevrotain/test/parse/grammar/follow_spec.ts index 7b5b68a53..7c41d23ee 100644 --- a/packages/chevrotain/test/parse/grammar/follow_spec.ts +++ b/packages/chevrotain/test/parse/grammar/follow_spec.ts @@ -11,7 +11,7 @@ import { Repetition, Rule, Terminal -} from "../../../src/parse/grammar/gast/gast_public" +} from "@chevrotain/gast" import keys from "lodash/keys" import { expect } from "chai" import { createToken } from "../../../src/scan/tokens_public" diff --git a/packages/chevrotain/test/parse/grammar/interperter_spec.ts b/packages/chevrotain/test/parse/grammar/interperter_spec.ts index d1c8dd882..96aef84c6 100644 --- a/packages/chevrotain/test/parse/grammar/interperter_spec.ts +++ b/packages/chevrotain/test/parse/grammar/interperter_spec.ts @@ -30,7 +30,7 @@ import { RepetitionMandatory, NonTerminal, RepetitionMandatoryWithSeparator -} from "../../../src/parse/grammar/gast/gast_public" +} from "@chevrotain/gast" import { createDeferredTokenBuilder } from "../../utils/builders" // ugly utilities to deffer execution of productive code until the relevant tests have started diff --git a/packages/chevrotain/test/parse/grammar/lookahead_spec.ts b/packages/chevrotain/test/parse/grammar/lookahead_spec.ts index 69b71652a..4b6e4d303 100644 --- a/packages/chevrotain/test/parse/grammar/lookahead_spec.ts +++ b/packages/chevrotain/test/parse/grammar/lookahead_spec.ts @@ -26,7 +26,7 @@ import { RepetitionWithSeparator, Rule, Terminal -} from "../../../src/parse/grammar/gast/gast_public" +} from "@chevrotain/gast" import { IToken, TokenType } from "@chevrotain/types" import { EmbeddedActionsParser } from "../../../src/parse/parser/traits/parser_traits" import { expect } from "chai" diff --git a/packages/chevrotain/test/parse/grammar/resolver_spec.ts b/packages/chevrotain/test/parse/grammar/resolver_spec.ts index 2dfd254bb..ff3e9830e 100644 --- a/packages/chevrotain/test/parse/grammar/resolver_spec.ts +++ b/packages/chevrotain/test/parse/grammar/resolver_spec.ts @@ -1,6 +1,6 @@ import { GastRefResolverVisitor } from "../../../src/parse/grammar/resolver" import { ParserDefinitionErrorType } from "../../../src/parse/parser/parser" -import { NonTerminal, Rule } from "../../../src/parse/grammar/gast/gast_public" +import { NonTerminal, Rule } from "@chevrotain/gast" import { defaultGrammarResolverErrorProvider } from "../../../src/parse/errors_public" import { expect } from "chai" diff --git a/packages/cst-dts-gen/api.d.ts b/packages/cst-dts-gen/api.d.ts deleted file mode 100644 index c821c104f..000000000 --- a/packages/cst-dts-gen/api.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { BaseParser } from "chevrotain" - -export declare function generateCstDts( - parser: BaseParser, - options?: GenerateDtsOptions -): string - -export declare type GenerateDtsOptions = { - includeTypes?: boolean - includeVisitorInterface?: boolean - visitorInterfaceName?: string -} diff --git a/packages/cst-dts-gen/package.json b/packages/cst-dts-gen/package.json index 4e4a0e764..6f5a59932 100644 --- a/packages/cst-dts-gen/package.json +++ b/packages/cst-dts-gen/package.json @@ -7,10 +7,10 @@ "url": "https://github.com/Chevrotain/chevrotain/issues" }, "license": "Apache-2.0", - "typings": "api.d.ts", + "typings": "lib/src/api.d.ts", "main": "lib/src/api.js", "files": [ - "api.d.ts", + "lib/src/**/*.ts", "lib/src/**/*.js", "lib/src/**/*.js.map" ], @@ -31,8 +31,9 @@ "coverage": "nyc mocha" }, "dependencies": { - "lodash": "4.17.21", - "chevrotain": "^9.1.0" + "@chevrotain/types": "^9.1.0", + "@chevrotain/gast": "^9.1.0", + "lodash": "4.17.21" }, "devDependencies": { "glob": "7.2.0" diff --git a/packages/cst-dts-gen/src/api.ts b/packages/cst-dts-gen/src/api.ts index dd954d55c..b1919ae07 100644 --- a/packages/cst-dts-gen/src/api.ts +++ b/packages/cst-dts-gen/src/api.ts @@ -1,24 +1,21 @@ -import { BaseParser } from "chevrotain" -import { GenerateDtsOptions } from "../api" +import { Rule, GenerateDtsOptions } from "@chevrotain/types" import { buildModel } from "./model" -import { genDts, GenDtsOptions } from "./generate" +import { genDts } from "./generate" -const defaultOptions: GenDtsOptions = { - includeTypes: true, +const defaultOptions: Required = { includeVisitorInterface: true, visitorInterfaceName: "ICstNodeVisitor" } export function generateCstDts( - parser: BaseParser, + productions: Record, options?: GenerateDtsOptions ): string { - const effectiveOptions: GenDtsOptions = { + const effectiveOptions = { ...defaultOptions, ...options } - const productions = parser.getGAstProductions() const model = buildModel(productions) return genDts(model, effectiveOptions) diff --git a/packages/cst-dts-gen/src/generate.ts b/packages/cst-dts-gen/src/generate.ts index 97cec7b4b..f00550957 100644 --- a/packages/cst-dts-gen/src/generate.ts +++ b/packages/cst-dts-gen/src/generate.ts @@ -1,36 +1,26 @@ import flatten from "lodash/flatten" import map from "lodash/map" import upperFirst from "lodash/upperFirst" - +import { GenerateDtsOptions } from "@chevrotain/types" import { CstNodeTypeDefinition, PropertyTypeDefinition, PropertyArrayType } from "./model" -export type GenDtsOptions = { - includeTypes: boolean - includeVisitorInterface: boolean - visitorInterfaceName: string -} - export function genDts( model: CstNodeTypeDefinition[], - options: GenDtsOptions + options: Required ): string { let contentParts: string[] = [] - if (options.includeTypes || options.includeVisitorInterface) { - contentParts = contentParts.concat( - `import type { CstNode, ICstVisitor, IToken } from "chevrotain";` - ) - } + contentParts = contentParts.concat( + `import type { CstNode, ICstVisitor, IToken } from "chevrotain";` + ) - if (options.includeTypes) { - contentParts = contentParts.concat( - flatten(map(model, (node) => genCstNodeTypes(node))) - ) - } + contentParts = contentParts.concat( + flatten(map(model, (node) => genCstNodeTypes(node))) + ) if (options.includeVisitorInterface) { contentParts = contentParts.concat( @@ -38,7 +28,7 @@ export function genDts( ) } - return contentParts.length ? contentParts.join("\n\n") + "\n" : "" + return contentParts.join("\n\n") + "\n" } function genCstNodeTypes(node: CstNodeTypeDefinition) { diff --git a/packages/cst-dts-gen/src/model.ts b/packages/cst-dts-gen/src/model.ts index aa6000e59..efcf1cc5a 100644 --- a/packages/cst-dts-gen/src/model.ts +++ b/packages/cst-dts-gen/src/model.ts @@ -1,9 +1,7 @@ -import { +import type { Alternation, Alternative, - GAstVisitor, IProduction, - NonTerminal, Option, Repetition, RepetitionMandatory, @@ -12,7 +10,8 @@ import { Rule, Terminal, TokenType -} from "chevrotain" +} from "@chevrotain/types" +import { NonTerminal, GAstVisitor } from "@chevrotain/gast" import map from "lodash/map" import flatten from "lodash/flatten" import values from "lodash/values" diff --git a/packages/cst-dts-gen/test/options_spec.ts b/packages/cst-dts-gen/test/options_spec.ts index 4fa9b396a..f67039216 100644 --- a/packages/cst-dts-gen/test/options_spec.ts +++ b/packages/cst-dts-gen/test/options_spec.ts @@ -1,21 +1,11 @@ -import { GenerateDtsOptions } from "../api" +import { GenerateDtsOptions } from "@chevrotain/types" import { generateCstDts } from "../src/api" import { CstParser, createToken } from "chevrotain" import { expect } from "chai" describe("The DTS generator", () => { - it("can generate nothing", () => { - const result = genDts({ - includeTypes: false, - includeVisitorInterface: false - }) - - expect(result).to.equal("") - }) - it("can generate only cst types", () => { const result = genDts({ - includeTypes: true, includeVisitorInterface: false }) @@ -24,20 +14,8 @@ describe("The DTS generator", () => { expect(result).to.include("export type TestRuleCstChildren") }) - it("can generate only cst visitor", () => { - const result = genDts({ - includeTypes: false, - includeVisitorInterface: true - }) - - expect(result).to.include("export interface ICstNodeVisitor") - expect(result).to.not.include("export interface TestRuleCstNode") - expect(result).to.not.include("export type TestRuleCstChildren") - }) - it("can generate a cst visitor with specific name", () => { const result = genDts({ - includeTypes: false, includeVisitorInterface: true, visitorInterfaceName: "ITestCstVisitor" }) @@ -48,7 +26,8 @@ describe("The DTS generator", () => { function genDts(options: GenerateDtsOptions) { const parser = new TestParser() - return generateCstDts(parser, options) + const productions = parser.getGAstProductions() + return generateCstDts(productions, options) } }) diff --git a/packages/cst-dts-gen/test/sample_test.ts b/packages/cst-dts-gen/test/sample_test.ts index 1bcbf617e..4ad09ace4 100644 --- a/packages/cst-dts-gen/test/sample_test.ts +++ b/packages/cst-dts-gen/test/sample_test.ts @@ -1,12 +1,13 @@ -import { generateCstDts } from "../src/api" import { BaseParser } from "chevrotain" import { expect } from "chai" import { readFileSync } from "fs" import { resolve, relative, basename } from "path" +import { generateCstDts } from "../src/api" -export function executeSampleTest(dirPath: string, parser: BaseParser) { +export function executeSampleTest(dirPath: string, parser: BaseParser): void { it("Can generate type definition", () => { - const result = generateCstDts(parser) + const productions = parser.getGAstProductions() + const result = generateCstDts(productions) const expectedOutputPath = getOutputFileForSnapshot(dirPath) const expectedOutput = readFileSync(expectedOutputPath).toString("utf8") const simpleNewLinesOutput = expectedOutput.replace(/\r\n/g, "\n") @@ -14,11 +15,11 @@ export function executeSampleTest(dirPath: string, parser: BaseParser) { }) } -export function testNameFromDir(dirPath: string) { +export function testNameFromDir(dirPath: string): string { return basename(dirPath) } -export function getOutputFileForSnapshot(libSnapshotDir: string) { +export function getOutputFileForSnapshot(libSnapshotDir: string): string { const srcSnapshotDir = getSourceFilePath(libSnapshotDir) return resolve(srcSnapshotDir, "output.d.ts") } @@ -27,7 +28,7 @@ export function getOutputFileForSnapshot(libSnapshotDir: string) { const packageDir = resolve(__dirname, "../..") const libDir = resolve(packageDir, "lib") -function getSourceFilePath(libFilePath: string) { +function getSourceFilePath(libFilePath: string): string { const relativeDirPath = relative(libDir, libFilePath) return resolve(packageDir, relativeDirPath) } diff --git a/packages/gast/.mocharc.js b/packages/gast/.mocharc.js new file mode 100644 index 000000000..5df43afd1 --- /dev/null +++ b/packages/gast/.mocharc.js @@ -0,0 +1,6 @@ +module.exports = { + recursive: true, + require: ["source-map-support/register"], + reporter: "spec", + spec: "./lib/test/**/*spec.js" +} diff --git a/packages/gast/nyc.config.js b/packages/gast/nyc.config.js new file mode 100644 index 000000000..1eb7de613 --- /dev/null +++ b/packages/gast/nyc.config.js @@ -0,0 +1 @@ +module.exports = require("../nyc.config") diff --git a/packages/gast/package.json b/packages/gast/package.json new file mode 100644 index 000000000..7b78c4176 --- /dev/null +++ b/packages/gast/package.json @@ -0,0 +1,42 @@ +{ + "name": "@chevrotain/gast", + "version": "9.1.0", + "description": "Grammar AST structure for Chevrotain Parsers", + "keywords": [], + "bugs": { + "url": "https://github.com/Chevrotain/chevrotain/issues" + }, + "license": "Apache-2.0", + "typings": "lib/src/api.d.ts", + "main": "lib/src/api.js", + "files": [ + "api.d.ts", + "lib/src/**/*.js", + "lib/src/**/*.js.map", + "lib/src/**/*.d.ts" + ], + "repository": { + "type": "git", + "url": "git://github.com/Chevrotain/chevrotain.git" + }, + "scripts": { + "---------- CI FLOWS --------": "", + "build": "npm-run-all clean compile", + "test": "npm-run-all coverage", + "---------- DEV FLOWS --------": "", + "update-snapshots": "node ./scripts/update-snapshots.js", + "---------- BUILD STEPS --------": "", + "clean": "shx rm -rf lib coverage", + "compile:watch": "tsc -w", + "compile": "tsc", + "coverage": "nyc mocha" + }, + "dependencies": { + "lodash": "4.17.21", + "@chevrotain/types": "^9.1.0" + }, + "devDependencies": {}, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/gast/src/api.ts b/packages/gast/src/api.ts new file mode 100644 index 000000000..6c8f4d4eb --- /dev/null +++ b/packages/gast/src/api.ts @@ -0,0 +1,23 @@ +export { + Rule, + Terminal, + NonTerminal, + Option, + Repetition, + RepetitionMandatory, + RepetitionMandatoryWithSeparator, + RepetitionWithSeparator, + Alternation, + Alternative, + serializeGrammar, + serializeProduction +} from "./model" + +export { GAstVisitor } from "./visitor" + +export { + getProductionDslName, + isOptionalProd, + isBranchingProd, + isSequenceProd +} from "./helpers" diff --git a/packages/gast/src/helpers.ts b/packages/gast/src/helpers.ts new file mode 100644 index 000000000..527940c08 --- /dev/null +++ b/packages/gast/src/helpers.ts @@ -0,0 +1,101 @@ +import some from "lodash/some" +import every from "lodash/every" +import has from "lodash/has" +import includes from "lodash/includes" +import { + AbstractProduction, + Alternation, + Alternative, + NonTerminal, + Option, + Repetition, + RepetitionMandatory, + RepetitionMandatoryWithSeparator, + RepetitionWithSeparator, + Rule, + Terminal +} from "./model" +import { GAstVisitor } from "./visitor" +import { IProduction, IProductionWithOccurrence } from "@chevrotain/types" + +export function isSequenceProd( + prod: IProduction +): prod is { definition: IProduction[] } & IProduction { + return ( + prod instanceof Alternative || + prod instanceof Option || + prod instanceof Repetition || + prod instanceof RepetitionMandatory || + prod instanceof RepetitionMandatoryWithSeparator || + prod instanceof RepetitionWithSeparator || + prod instanceof Terminal || + prod instanceof Rule + ) +} + +export function isOptionalProd( + prod: IProduction, + alreadyVisited: NonTerminal[] = [] +): boolean { + const isDirectlyOptional = + prod instanceof Option || + prod instanceof Repetition || + prod instanceof RepetitionWithSeparator + if (isDirectlyOptional) { + return true + } + + // note that this can cause infinite loop if one optional empty TOP production has a cyclic dependency with another + // empty optional top rule + // may be indirectly optional ((A?B?C?) | (D?E?F?)) + if (prod instanceof Alternation) { + // for OR its enough for just one of the alternatives to be optional + return some((prod).definition, (subProd: IProduction) => { + return isOptionalProd(subProd, alreadyVisited) + }) + } else if (prod instanceof NonTerminal && includes(alreadyVisited, prod)) { + // avoiding stack overflow due to infinite recursion + return false + } else if (prod instanceof AbstractProduction) { + if (prod instanceof NonTerminal) { + alreadyVisited.push(prod) + } + return every( + (prod).definition, + (subProd: IProduction) => { + return isOptionalProd(subProd, alreadyVisited) + } + ) + } else { + return false + } +} + +export function isBranchingProd( + prod: IProduction +): prod is { definition: IProduction[] } & IProduction { + return prod instanceof Alternation +} + +export function getProductionDslName(prod: IProductionWithOccurrence): string { + /* istanbul ignore else */ + if (prod instanceof NonTerminal) { + return "SUBRULE" + } else if (prod instanceof Option) { + return "OPTION" + } else if (prod instanceof Alternation) { + return "OR" + } else if (prod instanceof RepetitionMandatory) { + return "AT_LEAST_ONE" + } else if (prod instanceof RepetitionMandatoryWithSeparator) { + return "AT_LEAST_ONE_SEP" + } else if (prod instanceof RepetitionWithSeparator) { + return "MANY_SEP" + } else if (prod instanceof Repetition) { + return "MANY" + } else if (prod instanceof Terminal) { + return "CONSUME" + } else { + throw Error("non exhaustive match") + } +} diff --git a/packages/chevrotain/src/parse/grammar/gast/gast_public.ts b/packages/gast/src/model.ts similarity index 93% rename from packages/chevrotain/src/parse/grammar/gast/gast_public.ts rename to packages/gast/src/model.ts index 3b7a0eed3..e39c73c39 100644 --- a/packages/chevrotain/src/parse/grammar/gast/gast_public.ts +++ b/packages/gast/src/model.ts @@ -4,7 +4,6 @@ import isString from "lodash/isString" import isRegExp from "lodash/isRegExp" import pickBy from "lodash/pickBy" import assign from "lodash/assign" -import { tokenLabel } from "../../../scan/tokens_public" import { IGASTVisitor, IProduction, @@ -13,6 +12,22 @@ import { TokenType } from "@chevrotain/types" +// TODO: duplicated code to avoid extracting another sub-package -- how to avoid? +function tokenLabel(tokType: TokenType): string { + if (hasTokenLabel(tokType)) { + return tokType.LABEL + } else { + return tokType.name + } +} + +// TODO: duplicated code to avoid extracting another sub-package -- how to avoid? +function hasTokenLabel( + obj: TokenType +): obj is TokenType & Pick, "LABEL"> { + return isString(obj.LABEL) && obj.LABEL !== "" +} + export abstract class AbstractProduction implements IProduction { @@ -37,9 +52,9 @@ export class NonTerminal extends AbstractProduction implements IProductionWithOccurrence { - public nonTerminalName: string + public nonTerminalName!: string public label?: string - public referencedRule: Rule + public referencedRule!: Rule public idx: number = 1 constructor(options: { @@ -73,7 +88,7 @@ export class NonTerminal } export class Rule extends AbstractProduction { - public name: string + public name!: string public orgText: string = "" constructor(options: { @@ -148,7 +163,7 @@ export class RepetitionMandatoryWithSeparator extends AbstractProduction implements IProductionWithOccurrence { - public separator: TokenType + public separator!: TokenType public idx: number = 1 public maxLookahead?: number @@ -169,7 +184,7 @@ export class Repetition extends AbstractProduction implements IProductionWithOccurrence { - public separator: TokenType + public separator!: TokenType public idx: number = 1 public maxLookahead?: number @@ -190,7 +205,7 @@ export class RepetitionWithSeparator extends AbstractProduction implements IProductionWithOccurrence { - public separator: TokenType + public separator!: TokenType public idx: number = 1 public maxLookahead?: number @@ -239,7 +254,7 @@ export class Alternation } export class Terminal implements IProductionWithOccurrence { - public terminalType: TokenType + public terminalType!: TokenType public label?: string public idx: number = 1 diff --git a/packages/chevrotain/src/parse/grammar/gast/gast_visitor_public.ts b/packages/gast/src/visitor.ts similarity index 67% rename from packages/chevrotain/src/parse/grammar/gast/gast_visitor_public.ts rename to packages/gast/src/visitor.ts index 0b6e20383..b0b1c5f1a 100644 --- a/packages/chevrotain/src/parse/grammar/gast/gast_visitor_public.ts +++ b/packages/gast/src/visitor.ts @@ -9,7 +9,7 @@ import { RepetitionWithSeparator, Rule, Terminal -} from "./gast_public" +} from "./model" import { IProduction } from "@chevrotain/types" export abstract class GAstVisitor { @@ -42,25 +42,35 @@ export abstract class GAstVisitor { } } + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitNonTerminal(node: NonTerminal): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitAlternative(node: Alternative): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitOption(node: Option): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitRepetition(node: Repetition): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitRepetitionMandatory(node: RepetitionMandatory): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitRepetitionMandatoryWithSeparator( node: RepetitionMandatoryWithSeparator ): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitRepetitionWithSeparator(node: RepetitionWithSeparator): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitAlternation(node: Alternation): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitTerminal(node: Terminal): any {} + /* istanbul ignore next - testing the fact a NOOP function exists is non-trivial */ public visitRule(node: Rule): any {} } diff --git a/packages/gast/test/helpers_spec.ts b/packages/gast/test/helpers_spec.ts new file mode 100644 index 000000000..1e992c944 --- /dev/null +++ b/packages/gast/test/helpers_spec.ts @@ -0,0 +1,308 @@ +import { expect } from "chai" +import type { ITokenConfig, TokenType } from "@chevrotain/types" +import { + Alternation, + Terminal, + Rule, + Alternative, + Repetition, + RepetitionWithSeparator, + RepetitionMandatoryWithSeparator, + RepetitionMandatory, + Option, + NonTerminal, + isSequenceProd, + isOptionalProd, + isBranchingProd +} from "../src/api" + +function createDummyToken(opts: ITokenConfig): TokenType { + return { + name: opts.name, + PATTERN: opts.pattern + } +} + +describe("the gast helper utilities", () => { + let A: TokenType + let B: TokenType + + before(() => { + A = createDummyToken({ name: "A" }) + B = createDummyToken({ name: "B" }) + }) + + context("isSequenceProd()", () => { + context("positive for", () => { + it("Alternative", () => { + const prod = new Alternative({ definition: [] }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("Option", () => { + const prod = new Option({ definition: [] }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("Option", () => { + const prod = new Option({ definition: [] }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("Repetition", () => { + const prod = new Repetition({ definition: [] }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("RepetitionMandatory", () => { + const prod = new RepetitionMandatory({ definition: [] }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("RepetitionMandatoryWithSeparator", () => { + const prod = new RepetitionMandatoryWithSeparator({ + definition: [], + separator: A + }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("RepetitionWithSeparator", () => { + const prod = new RepetitionWithSeparator({ + definition: [], + separator: A + }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("Terminal", () => { + const prod = new Terminal({ + terminalType: A + }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + + it("Rule", () => { + const prod = new Rule({ name: "foo", definition: [] }) + const result = isSequenceProd(prod) + expect(result).to.be.true + }) + }) + + context("negative for", () => { + it("NonTerminal", () => { + const prod = new NonTerminal({ nonTerminalName: "bar" }) + const result = isSequenceProd(prod) + expect(result).to.be.false + }) + + it("Alternation", () => { + const prod = new Alternation({ definition: [] }) + const result = isSequenceProd(prod) + expect(result).to.be.false + }) + }) + }) + + context("isOptionalProd()", () => { + context("positive for", () => { + it("an Alternation where some alternative is empty", () => { + const prod = new Alternation({ + definition: [ + new Alternative({ + definition: [new Terminal({ terminalType: A })] + }), + new Alternative({ definition: [] }) + ] + }) + const result = isOptionalProd(prod) + expect(result).to.be.true + }) + + it("Option", () => { + const prod = new Option({ definition: [] }) + const result = isOptionalProd(prod) + expect(result).to.be.true + }) + + it("Repetition", () => { + const prod = new Repetition({ definition: [] }) + const result = isOptionalProd(prod) + expect(result).to.be.true + }) + + it("RepetitionWithSeparator", () => { + const prod = new RepetitionWithSeparator({ + definition: [], + separator: A + }) + const result = isOptionalProd(prod) + expect(result).to.be.true + }) + }) + + context("negative for", () => { + it("an Alternation where every alternative is non-empty", () => { + const prod = new Alternation({ + definition: [ + new Alternative({ + definition: [new Terminal({ terminalType: A })] + }), + new Alternative({ definition: [new Terminal({ terminalType: B })] }) + ] + }) + const result = isOptionalProd(prod) + expect(result).to.be.false + }) + + it("non empty Alternative", () => { + const prod = new Alternative({ + definition: [new Terminal({ terminalType: A })] + }) + const result = isOptionalProd(prod) + expect(result).to.be.false + }) + + it("NonTerminal", () => { + const prod = new NonTerminal({ nonTerminalName: "bar" }) + const result = isSequenceProd(prod) + expect(result).to.be.false + }) + + it("non-empty RepetitionMandatory", () => { + const prod = new RepetitionMandatory({ + definition: [new Terminal({ terminalType: A })] + }) + const result = isOptionalProd(prod) + expect(result).to.be.false + }) + + it("non-empty RepetitionMandatoryWithSeparator", () => { + const prod = new RepetitionMandatoryWithSeparator({ + definition: [new Terminal({ terminalType: A })], + separator: A + }) + const result = isOptionalProd(prod) + expect(result).to.be.false + }) + + it("Terminal", () => { + const prod = new Terminal({ + terminalType: A + }) + const result = isOptionalProd(prod) + expect(result).to.be.false + }) + + it("none empty Rule", () => { + const prod = new Rule({ + name: "foo", + definition: [new Terminal({ terminalType: A })] + }) + const result = isOptionalProd(prod) + expect(result).to.be.false + }) + }) + + context("edge case", () => { + it("will avoid infinite loop on recursive Non-Terminals", () => { + const recursiveRule = new Rule({ name: "recursive", definition: [] }) + const recursiveNonTerminal = new NonTerminal({ + nonTerminalName: "recursive", + referencedRule: recursiveRule + }) + recursiveRule.definition = [recursiveNonTerminal] + const result = isOptionalProd(recursiveNonTerminal) + expect(result).to.be.false + }) + }) + }) + + context("isBranchingProd()", () => { + context("positive for", () => { + it("Alternation", () => { + const prod = new Alternation({ definition: [] }) + const result = isBranchingProd(prod) + expect(result).to.be.true + }) + }) + + context("negative for", () => { + it("NonTerminal", () => { + const prod = new NonTerminal({ nonTerminalName: "bar" }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("Alternative", () => { + const prod = new Alternative({ definition: [] }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("Option", () => { + const prod = new Option({ definition: [] }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("Option", () => { + const prod = new Option({ definition: [] }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("Repetition", () => { + const prod = new Repetition({ definition: [] }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("RepetitionMandatory", () => { + const prod = new RepetitionMandatory({ definition: [] }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("RepetitionMandatoryWithSeparator", () => { + const prod = new RepetitionMandatoryWithSeparator({ + definition: [], + separator: A + }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("RepetitionWithSeparator", () => { + const prod = new RepetitionWithSeparator({ + definition: [], + separator: A + }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("Terminal", () => { + const prod = new Terminal({ + terminalType: A + }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + + it("Rule", () => { + const prod = new Rule({ name: "foo", definition: [] }) + const result = isBranchingProd(prod) + expect(result).to.be.false + }) + }) + }) +}) diff --git a/packages/chevrotain/test/parse/grammar/gast_spec.ts b/packages/gast/test/model_spec.ts similarity index 95% rename from packages/chevrotain/test/parse/grammar/gast_spec.ts rename to packages/gast/test/model_spec.ts index 5c4fe7506..c07fa7252 100644 --- a/packages/chevrotain/test/parse/grammar/gast_spec.ts +++ b/packages/gast/test/model_spec.ts @@ -1,6 +1,7 @@ -import { getProductionDslName } from "../../../src/parse/grammar/gast/gast" -import { createToken } from "../../../src/scan/tokens_public" +import { expect } from "chai" +import { ITokenConfig, TokenType } from "@chevrotain/types" import { + getProductionDslName, Alternation, NonTerminal, RepetitionMandatory, @@ -13,11 +14,16 @@ import { Alternative, Rule, serializeGrammar -} from "../../../src/parse/grammar/gast/gast_public" -import { expect } from "chai" -import { TokenType } from "@chevrotain/types" +} from "../src/api" + +function createDummyToken(opts: ITokenConfig): TokenType { + return { + name: opts.name, + PATTERN: opts.pattern + } +} -describe("GAst namespace", () => { +describe("the gast model", () => { describe("the ProdRef class", () => { it("will always return a valid empty definition, even if it's ref is unresolved", () => { const prodRef = new NonTerminal({ @@ -101,13 +107,13 @@ describe("GAst namespace", () => { let WithLiteral: TokenType before(() => { - A = createToken({ name: "A" }) + A = createDummyToken({ name: "A" }) A.LABEL = "bamba" - B = createToken({ name: "B", pattern: /[a-zA-Z]\w*/ }) - C = createToken({ name: "C" }) - D = createToken({ name: "D" }) - Comma = createToken({ name: "Comma" }) - WithLiteral = createToken({ + B = createDummyToken({ name: "B", pattern: /[a-zA-Z]\w*/ }) + C = createDummyToken({ name: "C" }) + D = createDummyToken({ name: "D" }) + Comma = createDummyToken({ name: "Comma" }) + WithLiteral = createDummyToken({ name: "WithLiteral", pattern: "bamba" }) diff --git a/packages/gast/test/visitor_spec.ts b/packages/gast/test/visitor_spec.ts new file mode 100644 index 000000000..9a5c0315d --- /dev/null +++ b/packages/gast/test/visitor_spec.ts @@ -0,0 +1,249 @@ +import { expect } from "chai" +import type { ITokenConfig, TokenType } from "@chevrotain/types" +import { + GAstVisitor, + Terminal, + Rule, + Alternation, + Alternative, + Repetition, + RepetitionWithSeparator, + RepetitionMandatoryWithSeparator, + RepetitionMandatory, + Option, + NonTerminal +} from "../src/api" + +function createDummyToken(opts: ITokenConfig): TokenType { + return { + name: opts.name, + PATTERN: opts.pattern + } +} + +describe("the gast visitor", () => { + context("visit/traversal methods", () => { + let A: TokenType + let B: TokenType + + before(() => { + A = createDummyToken({ name: "A" }) + B = createDummyToken({ name: "B" }) + }) + + it("can visit a terminal", () => { + let visited = false + const terminal = new Terminal({ terminalType: A }) + + class TerminalTestVisitor extends GAstVisitor { + visitTerminal(node: Terminal): void { + expect(node).to.equal(terminal) + visited = true + } + } + + const visitor = new TerminalTestVisitor() + terminal.accept(visitor) + expect(visited).to.be.true + }) + + it("can visit a Rule", () => { + let visitedChild = false + let visitedTop = false + const ruleNode = new Rule({ + name: "foo", + definition: [new Terminal({ terminalType: B })] + }) + + class TestVisitor extends GAstVisitor { + visitRule(node: Rule) { + expect(node).to.equal(ruleNode) + visitedTop = true + } + visitTerminal(node: Terminal): void { + expect(node.terminalType).to.equal(B) + visitedChild = true + } + } + + const visitor = new TestVisitor() + ruleNode.accept(visitor) + expect(visitedTop).to.be.true + expect(visitedChild).to.be.true + }) + + it("can visit an Alternation", () => { + const visitedChild: boolean[] = [] + let visitedTop = false + const alternation = new Alternation({ + definition: [ + new Alternative({ definition: [new Terminal({ terminalType: B })] }) + ] + }) + + class TestVisitor extends GAstVisitor { + visitAlternative(node: Alternative): any { + visitedChild.push(true) + } + + visitTerminal(node: Terminal): void { + expect(node.terminalType).to.equal(B) + visitedChild.push(true) + } + visitAlternation(node: Alternation): void { + expect(node).to.equal(alternation) + visitedTop = true + } + } + + const visitor = new TestVisitor() + alternation.accept(visitor) + expect(visitedTop).to.be.true + expect(visitedChild).to.deep.equal([true, true]) + }) + + it("can visit a Repetition", () => { + let visitedChild = false + let visitedRoot = false + const rootNode = new Repetition({ + definition: [new Terminal({ terminalType: B })] + }) + + class TestVisitor extends GAstVisitor { + visitRepetition(node: Repetition) { + expect(node).to.equal(rootNode) + visitedRoot = true + } + visitTerminal(node: Terminal): void { + expect(node.terminalType).to.equal(B) + visitedChild = true + } + } + + const visitor = new TestVisitor() + rootNode.accept(visitor) + expect(visitedRoot).to.be.true + expect(visitedChild).to.be.true + }) + + it("can visit a Repetition Mandatory", () => { + let visitedChild = false + let visitedRoot = false + const rootNode = new RepetitionMandatory({ + definition: [new Terminal({ terminalType: B })] + }) + + class TestVisitor extends GAstVisitor { + visitRepetitionMandatory(node: Repetition) { + expect(node).to.equal(rootNode) + visitedRoot = true + } + visitTerminal(node: Terminal): void { + expect(node.terminalType).to.equal(B) + visitedChild = true + } + } + + const visitor = new TestVisitor() + rootNode.accept(visitor) + expect(visitedRoot).to.be.true + expect(visitedChild).to.be.true + }) + + it("can visit a Repetition With Separator", () => { + let visitedChild = false + let visitedRoot = false + const rootNode = new RepetitionWithSeparator({ + separator: A, + definition: [new Terminal({ terminalType: B })] + }) + + class TestVisitor extends GAstVisitor { + visitRepetitionWithSeparator(node: RepetitionWithSeparator) { + expect(node).to.equal(rootNode) + expect(node.separator).to.equal + visitedRoot = true + } + visitTerminal(node: Terminal): void { + expect(node.terminalType).to.equal(B) + visitedChild = true + } + } + + const visitor = new TestVisitor() + rootNode.accept(visitor) + expect(visitedRoot).to.be.true + expect(visitedChild).to.be.true + }) + + it("can visit a Repetition Mandatory With Separator", () => { + let visitedChild = false + let visitedRoot = false + const rootNode = new RepetitionMandatoryWithSeparator({ + separator: A, + definition: [new Terminal({ terminalType: B })] + }) + + class TestVisitor extends GAstVisitor { + visitRepetitionMandatoryWithSeparator( + node: RepetitionMandatoryWithSeparator + ): void { + expect(node).to.equal(rootNode) + expect(node.separator).to.equal + visitedRoot = true + } + + visitTerminal(node: Terminal): void { + expect(node.terminalType).to.equal(B) + visitedChild = true + } + } + + const visitor = new TestVisitor() + rootNode.accept(visitor) + expect(visitedRoot).to.be.true + expect(visitedChild).to.be.true + }) + + it("can visit an Option", () => { + let visitedChild = false + let visitedRoot = false + const rootNode = new Option({ + definition: [new Terminal({ terminalType: B })] + }) + + class TestVisitor extends GAstVisitor { + visitOption(node: Option): void { + expect(node).to.equal(rootNode) + visitedRoot = true + } + + visitTerminal(node: Terminal): void { + expect(node.terminalType).to.equal(B) + visitedChild = true + } + } + + const visitor = new TestVisitor() + rootNode.accept(visitor) + expect(visitedRoot).to.be.true + expect(visitedChild).to.be.true + }) + + it("can visit an Non-Terminal", () => { + let visitedRoot = false + const rootNode = new NonTerminal({ nonTerminalName: "foo" }) + + class TestVisitor extends GAstVisitor { + visitNonTerminal(node: NonTerminal): any { + expect(node.nonTerminalName).to.equal("foo") + visitedRoot = true + } + } + + const visitor = new TestVisitor() + rootNode.accept(visitor) + expect(visitedRoot).to.be.true + }) + }) +}) diff --git a/packages/gast/tsconfig.json b/packages/gast/tsconfig.json new file mode 100644 index 000000000..1539875cc --- /dev/null +++ b/packages/gast/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "lib", + "baseUrl": "." + }, + "include": ["./src/**/*", "./test/**/*", "api.d.ts"] +} diff --git a/packages/types/api.d.ts b/packages/types/api.d.ts index b1c87137c..a57690c2c 100644 --- a/packages/types/api.d.ts +++ b/packages/types/api.d.ts @@ -2560,3 +2560,36 @@ export declare function createSyntaxDiagramsCode( grammar: ISerializedGast[], config?: ICreateSyntaxDiagramsConfig ): string + +/** + * Will generate TypeScript definitions source code (text). + * For a set of {@link Rule}. + * + * This set of Rules can be obtained from a Parser **instance** via the + * {@link BaseParser.getGAstProductions} method. + * + * Note that this function produces a **string** the output. + * It is the responsibility of the end-user to create the signatures files. + * - e.g: via `fs.writeFileSync()` + * + * See: https://chevrotain.io/docs/guide/concrete_syntax_tree.html##cst-typescript-signatures + */ +export declare function generateCstDts( + productions: Record, + options?: GenerateDtsOptions +): string + +export declare type GenerateDtsOptions = { + /** + * `true` by default. + * Disable this to prevent the generation of the CSTVisitor interface. + * For example, if a different traversal method on the CST has been implemented + * by the end-user and the Chevrotain CST Visitor apis are not used. + */ + includeVisitorInterface?: boolean + /** + * The generated visitor interface will be called `ICstNodeVisitor` by default + * This parameter enables giving it a more specific name, for example: `MyCstVisitor` or `JohnDoe` + */ + visitorInterfaceName?: string +} diff --git a/packages/website/docs/guide/concrete_syntax_tree.md b/packages/website/docs/guide/concrete_syntax_tree.md index a4cc5fee5..82985c8be 100644 --- a/packages/website/docs/guide/concrete_syntax_tree.md +++ b/packages/website/docs/guide/concrete_syntax_tree.md @@ -196,8 +196,8 @@ $.RULE("statements", () => { }) ``` -Some of the Terminals and Non-Terminals are used in **both** alternatives. -It is possible to check for the existence of distinguishing terminals such as the "Let" and "Select". +Some Terminals and Non-Terminals are used in **both** alternatives. +It is possible to check for the existence of "distinguishing" terminals such as the "Let" and "Select". But this is not a robust approach. ```javascript @@ -231,7 +231,7 @@ will become too difficult to understand and maintain due to verbosity. Sometimes the information regarding the textual location (range) of each CstNode is needed. This information is normally **already present** on the CstNodes **nested** children simply because the CstNode's children -include the Tokens provided by the Lexer. However by default this information is not easily accessible +include the Tokens provided by the Lexer. However, by default this information is not easily accessible as we would have to fully traverse a CstNode to understands its full location range information. The feature for providing CstNode location directly on the CstNodes objects is available since version 4.7.0. @@ -273,16 +273,16 @@ Caveats - This feature has a slight performance and memory cost, this performance impact is **linear** and was measured at 5-10% for a full lexing + parsing flow. In general the more complex a grammar is (in terms of more CstNodes created per N tokens) - the higher the impact. Additionally if the Parser has activated the error recovery capabilities + the higher the impact. Additionally, if the Parser has activated the error recovery capabilities of Chevrotain the impact would be at the high end of the given range, - as the location tracking logic is more complex when some of the Tokens may be virtual/invalid. + as the location tracking logic is more complex when some Tokens may be virtual/invalid. ## Fault Tolerance CST output is also supported in combination with automatic error recovery. This combination is actually stronger than regular error recovery because even partially formed CstNodes will be present on the CST output and be marked -using the **recoveredNode"** boolean property. +using the `recoveredNode` boolean property. For example given this grammar and assuming the parser re-synced after a token mismatch at the "Where" token: @@ -321,7 +321,7 @@ for example: offering auto-fix suggestions or provide better error messages. ## Traversing -So we now know how to create a CST and it's internal structure. +So, we now know how to create a CST, and it's internal structure. But how do we traverse this structure and perform semantic actions? Some examples for such semantic actions: @@ -376,7 +376,7 @@ This is a valid approach, however it can be somewhat error prone: For the impatient, See a full runnable example: [Calculator Grammar with CSTVisitor interpreter](https://github.com/chevrotain/chevrotain/blob/master/examples/grammars/calculator/calculator_pure_grammar.js) -Chevrotain provides a CSTVisitor class which can make traversing the CST less error prone. +Chevrotain provides a CSTVisitor class which can make traversing the CST less error-prone. ```javascript // The base Visitor Class can be accessed via a Parser **instance**. @@ -465,6 +465,58 @@ It is not possible to return values from the visit methods because the default implementation does not return any value, only traverses the CST thus the chain of returned values will be broken. +## Generating TypeScript Signatures(d.ts) for CST constructs. + +In the sections above we have seen that implementing a Chevrotain `CstParser` would also **implicitly** define +several data structures and APIs: + +1. A CSTNode for each grammar rule. +2. A CST-Visitor API for the whole set of rules + +But what if we want **explicit** definitions for these data structures and APIs? + +- For example to easily implement our CST Visitors in TypeScript instead of over-using the `any` type... + +This capability is provided via the [generateCstDts](https://chevrotain.io/documentation/9_1_0/modules.html#generateCstDts) function. +Which given a set of grammar `Rules` will generate the **source text** for the corresponding TypeScript signatures. + +For example, given the Parser rules for **arrays** in JSON. + +```typescript +class JSONParser extends CstParser { + private array = this.RULE("array", () => { + this.CONSUME(LSquare) + this.MANY_SEP({ + SEP: Comma, + DEF: () => { + this.SUBRULE(this.value) + } + }) + this.CONSUME(RSquare) + }) +} +``` + +It would produce the following signatures: + +```typescript +export interface ArrayCstNode extends CstNode { + name: "array" + children: ArrayCstChildren +} + +export type ArrayCstChildren = { + LSquare: IToken[] + value?: ValueCstNode[] + Comma?: IToken[] + RSquare: IToken[] +} +``` + +Note that the [generateCstDts](https://chevrotain.io/documentation/9_1_0/modules.html#generateCstDts) function +only produces the **source text** of the TypeScript signatures, and it is the end-user's responsibility to save +the contents to a file, see: minimal [generation script example](https://github.com/Chevrotain/chevrotain/tree/master/examples/implementation_languages/typescript/scripts/gen_dts_signatures.js). + ## Performance On V8 (Chrome/Node) building the CST was measured at anywhere from 35%-90% of the performance @@ -482,6 +534,6 @@ This may be substantial yet please consider: - Parsing is usually just one step in a larger flow, so the overall impact even in the slower edge cases would be reduced. -It is therefore recommended to use the CST creation capabilities +It is therefore recommended using the CST creation capabilities as its benefits (modularity / ease of maintenance) by far outweigh the costs (potentially reduced performance). except in unique edge cases. diff --git a/tsconfig.json b/tsconfig.json index 892e532fa..a746fcf7c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,10 +7,13 @@ "path": "./packages/types" }, { - "path": "./packages/cst-dts-gen" + "path": "./packages/gast" }, { "path": "./packages/chevrotain" + }, + { + "path": "./packages/cst-dts-gen" } ], // needed to avoid trying to compile other .ts files in this repo