diff --git a/examples/parser/parametrized_rules/parametrized.js b/examples/parser/parametrized_rules/parametrized.js index 2b7a8bacc..3f58df1fa 100644 --- a/examples/parser/parametrized_rules/parametrized.js +++ b/examples/parser/parametrized_rules/parametrized.js @@ -110,11 +110,7 @@ module.exports = function (text, mood) { // setting a new input will RESET the parser instance's state. parser.input = lexResult.tokens - // Passing the argument to the top rule. - // note that because we are invoking a "start rule" we must provide the arguments as the second parameter. - // with the first parameter provided the value <1> - // also note that the arguments are passed as an array - const cst = parser.topRule(1, [mood]) + const cst = parser.topRule(mood) return { cst: cst, diff --git a/packages/chevrotain/src/lang/lang_extensions.ts b/packages/chevrotain/src/lang/lang_extensions.ts index 602ccfa89..520656dc9 100644 --- a/packages/chevrotain/src/lang/lang_extensions.ts +++ b/packages/chevrotain/src/lang/lang_extensions.ts @@ -1,6 +1,6 @@ const NAME = "name" -export function defineNameProp(obj, nameValue): void { +export function defineNameProp(obj: {}, nameValue: string): void { Object.defineProperty(obj, NAME, { enumerable: false, configurable: true, diff --git a/packages/chevrotain/src/parse/cst/cst_visitor.ts b/packages/chevrotain/src/parse/cst/cst_visitor.ts index ac307cf0e..38185b362 100644 --- a/packages/chevrotain/src/parse/cst/cst_visitor.ts +++ b/packages/chevrotain/src/parse/cst/cst_visitor.ts @@ -10,7 +10,7 @@ import { map } from "@chevrotain/utils" import { defineNameProp } from "../../lang/lang_extensions" -import { ICstVisitor } from "@chevrotain/types" +import { CstNode, ICstVisitor } from "@chevrotain/types" export function defaultVisit(ctx: any, param: IN): OUT { const childrenNames = keys(ctx) @@ -45,7 +45,7 @@ export function createBaseSemanticVisitorConstructor( defineNameProp(derivedConstructor, grammarName + "BaseSemantics") const semanticProto = { - visit: function (cstNode, param) { + visit: function (cstNode: CstNode | CstNode[], param: any) { // enables writing more concise visitor methods when CstNode has only a single child if (isArray(cstNode)) { // A CST Node's children dictionary can never have empty arrays as values @@ -121,7 +121,7 @@ export interface IVisitorDefinitionError { } export function validateVisitor( - visitorInstance: Function, + visitorInstance: ICstVisitor, ruleNames: string[] ): IVisitorDefinitionError[] { const missingErrors = validateMissingCstMethods(visitorInstance, ruleNames) @@ -131,11 +131,11 @@ export function validateVisitor( } export function validateMissingCstMethods( - visitorInstance: Function, + visitorInstance: ICstVisitor, ruleNames: string[] ): IVisitorDefinitionError[] { const errors: IVisitorDefinitionError[] = map(ruleNames, (currRuleName) => { - if (!isFunction(visitorInstance[currRuleName])) { + if (!isFunction((visitorInstance as any)[currRuleName])) { return { msg: `Missing visitor method: <${currRuleName}> on ${( visitorInstance.constructor.name @@ -151,14 +151,14 @@ export function validateMissingCstMethods( const VALID_PROP_NAMES = ["constructor", "visit", "validateVisitor"] export function validateRedundantMethods( - visitorInstance: Function, + visitorInstance: ICstVisitor, ruleNames: string[] ): IVisitorDefinitionError[] { const errors = [] for (const prop in visitorInstance) { if ( - isFunction(visitorInstance[prop]) && + isFunction((visitorInstance as any)[prop]) && !contains(VALID_PROP_NAMES, prop) && !contains(ruleNames, prop) ) { diff --git a/packages/chevrotain/src/parse/exceptions_public.ts b/packages/chevrotain/src/parse/exceptions_public.ts index 8cf437afc..b0937b390 100644 --- a/packages/chevrotain/src/parse/exceptions_public.ts +++ b/packages/chevrotain/src/parse/exceptions_public.ts @@ -30,7 +30,7 @@ abstract class RecognitionException implements IRecognitionException { context: IRecognizerContext - resyncedTokens = [] + resyncedTokens: IToken[] = [] protected constructor(message: string, public token: IToken) { super(message) diff --git a/packages/chevrotain/src/parse/grammar/checks.ts b/packages/chevrotain/src/parse/grammar/checks.ts index 863ca33fa..decd2517b 100644 --- a/packages/chevrotain/src/parse/grammar/checks.ts +++ b/packages/chevrotain/src/parse/grammar/checks.ts @@ -1,8 +1,8 @@ import * as utils from "@chevrotain/utils" import { contains, - every, findAll, + flatMap, flatten, forEach, isEmpty, @@ -56,24 +56,24 @@ export function validateGrammar( errMsgProvider: IGrammarValidatorErrorMessageProvider, grammarName: string ): IParserDefinitionError[] { - const duplicateErrors: any = utils.map(topLevels, (currTopLevel) => + const duplicateErrors = flatMap(topLevels, (currTopLevel) => validateDuplicateProductions(currTopLevel, errMsgProvider) ) - const leftRecursionErrors: any = utils.map(topLevels, (currTopRule) => + const leftRecursionErrors = flatMap(topLevels, (currTopRule) => validateNoLeftRecursion(currTopRule, currTopRule, errMsgProvider) ) - let emptyAltErrors = [] - let ambiguousAltsErrors = [] - let emptyRepetitionErrors = [] + let emptyAltErrors: IParserEmptyAlternativeDefinitionError[] = [] + let ambiguousAltsErrors: IParserAmbiguousAlternativesDefinitionError[] = [] + let emptyRepetitionErrors: IParserDefinitionError[] = [] // left recursion could cause infinite loops in the following validations. // It is safest to first have the user fix the left recursion errors first and only then examine Further issues. - if (every(leftRecursionErrors, isEmpty)) { - emptyAltErrors = map(topLevels, (currTopRule) => + if (isEmpty(leftRecursionErrors)) { + emptyAltErrors = flatMap(topLevels, (currTopRule) => validateEmptyOrAlternative(currTopRule, errMsgProvider) ) - ambiguousAltsErrors = map(topLevels, (currTopRule) => + ambiguousAltsErrors = flatMap(topLevels, (currTopRule) => validateAmbiguousAlternationAlternatives( currTopRule, globalMaxLookahead, @@ -94,11 +94,11 @@ export function validateGrammar( errMsgProvider ) - const tooManyAltsErrors = map(topLevels, (curRule) => + const tooManyAltsErrors = flatMap(topLevels, (curRule) => validateTooManyAlts(curRule, errMsgProvider) ) - const duplicateRulesError = map(topLevels, (curRule) => + const duplicateRulesError = flatMap(topLevels, (curRule) => validateRuleDoesNotAlreadyExist( curRule, topLevels, @@ -107,18 +107,14 @@ export function validateGrammar( ) ) - return ( - utils.flatten( - duplicateErrors.concat( - emptyRepetitionErrors, - leftRecursionErrors, - emptyAltErrors, - ambiguousAltsErrors, - termsNamespaceConflictErrors, - tooManyAltsErrors, - duplicateRulesError - ) - ) + return (duplicateErrors as IParserDefinitionError[]).concat( + emptyRepetitionErrors, + leftRecursionErrors, + emptyAltErrors, + ambiguousAltsErrors, + termsNamespaceConflictErrors, + tooManyAltsErrors, + duplicateRulesError ) } @@ -223,7 +219,7 @@ export class OccurrenceValidationCollector extends GAstVisitor { export function validateRuleDoesNotAlreadyExist( rule: Rule, allRules: Rule[], - className, + className: string, errMsgProvider: IGrammarValidatorErrorMessageProvider ): IParserDefinitionError[] { const errors = [] @@ -258,7 +254,7 @@ export function validateRuleDoesNotAlreadyExist( export function validateRuleIsOverridden( ruleName: string, definedRulesNames: string[], - className + className: string ): IParserDefinitionError[] { const errors = [] let errMsg @@ -283,7 +279,7 @@ export function validateNoLeftRecursion( errMsgProvider: IGrammarValidatorErrorMessageProvider, path: Rule[] = [] ): IParserDefinitionError[] { - const errors = [] + const errors: IParserDefinitionError[] = [] const nextNonTerminals = getFirstNoneTerminal(currRule.definition) if (utils.isEmpty(nextNonTerminals)) { return [] @@ -307,7 +303,7 @@ export function validateNoLeftRecursion( nextNonTerminals, path.concat([topRule]) ) - const errorsFromNextSteps = utils.map(validNextSteps, (currRefRule) => { + const errorsFromNextSteps = flatMap(validNextSteps, (currRefRule) => { const newPath = utils.cloneArr(path) newPath.push(currRefRule) return validateNoLeftRecursion( @@ -318,12 +314,12 @@ export function validateNoLeftRecursion( ) }) - return errors.concat(utils.flatten(errorsFromNextSteps)) + return errors.concat(errorsFromNextSteps) } } export function getFirstNoneTerminal(definition: IProduction[]): Rule[] { - let result = [] + let result: Rule[] = [] if (utils.isEmpty(definition)) { return result } @@ -367,7 +363,7 @@ export function getFirstNoneTerminal(definition: IProduction[]): Rule[] { } class OrCollector extends GAstVisitor { - public alternations = [] + public alternations: Alternation[] = [] public visitAlternation(node: Alternation): void { this.alternations.push(node) @@ -524,7 +520,7 @@ export function validateSomeNonEmptyLookaheadPath( maxLookahead: number, errMsgProvider: IGrammarValidatorErrorMessageProvider ): IParserDefinitionError[] { - const errors = [] + const errors: IParserDefinitionError[] = [] forEach(topLevelRules, (currTopRule) => { const collectorVisitor = new RepetitionCollector() currTopRule.accept(collectorVisitor) @@ -568,7 +564,7 @@ function checkAlternativesAmbiguities( rule: Rule, errMsgProvider: IGrammarValidatorErrorMessageProvider ): IParserAmbiguousAlternativesDefinitionError[] { - const foundAmbiguousPaths = [] + const foundAmbiguousPaths: Alternative = [] const identicalAmbiguities = reduce( alternatives, (result, currAlt, currAltIdx) => { @@ -636,9 +632,7 @@ export function checkPrefixAlternativesAmbiguities( alternation: Alternation, rule: Rule, errMsgProvider: IGrammarValidatorErrorMessageProvider -): IAmbiguityDescriptor[] { - let errors = [] - +): IParserAmbiguousAlternativesDefinitionError[] { // flatten const pathsAndIndices = reduce( alternatives, @@ -648,10 +642,10 @@ export function checkPrefixAlternativesAmbiguities( }) return result.concat(currPathsAndIdx) }, - [] + [] as { idx: number; path: TokenType[] }[] ) - forEach(pathsAndIndices, (currPathAndIdx) => { + const errors = flatMap(pathsAndIndices, (currPathAndIdx) => { const alternativeGast = alternation.definition[currPathAndIdx.idx] // ignore (skip) ambiguities with this alternative if (alternativeGast.ignoreAmbiguities === true) { @@ -678,7 +672,7 @@ export function checkPrefixAlternativesAmbiguities( const currPathPrefixErrors = map( prefixAmbiguitiesPathsAndIndices, - (currAmbPathAndIdx) => { + (currAmbPathAndIdx): IParserAmbiguousAlternativesDefinitionError => { const ambgIndices = [currAmbPathAndIdx.idx + 1, targetIdx + 1] const occurrence = alternation.idx === 0 ? "" : alternation.idx @@ -697,7 +691,8 @@ export function checkPrefixAlternativesAmbiguities( } } ) - errors = errors.concat(currPathPrefixErrors) + + return currPathPrefixErrors }) return errors @@ -708,7 +703,7 @@ function checkTerminalAndNoneTerminalsNameSpace( tokenTypes: TokenType[], errMsgProvider: IGrammarValidatorErrorMessageProvider ): IParserDefinitionError[] { - const errors = [] + const errors: IParserDefinitionError[] = [] const tokenNames = map(tokenTypes, (currToken) => currToken.name) diff --git a/packages/chevrotain/src/parse/grammar/follow.ts b/packages/chevrotain/src/parse/grammar/follow.ts index b55ddf01d..8a233606e 100644 --- a/packages/chevrotain/src/parse/grammar/follow.ts +++ b/packages/chevrotain/src/parse/grammar/follow.ts @@ -8,7 +8,7 @@ import { IProduction, TokenType } from "@chevrotain/types" // This ResyncFollowsWalker computes all of the follows required for RESYNC // (skipping reference production). export class ResyncFollowsWalker extends RestWalker { - public follows = {} + public follows: Record = {} constructor(private topProd: Rule) { super() diff --git a/packages/chevrotain/src/parse/grammar/gast/gast.ts b/packages/chevrotain/src/parse/grammar/gast/gast.ts index 25058f701..04cb07949 100644 --- a/packages/chevrotain/src/parse/grammar/gast/gast.ts +++ b/packages/chevrotain/src/parse/grammar/gast/gast.ts @@ -96,7 +96,15 @@ export function getProductionDslName(prod: IProductionWithOccurrence): string { export class DslMethodsCollectorVisitor extends GAstVisitor { // A minus is never valid in an identifier name public separator = "-" - public dslMethods = { + public dslMethods: { + [subruleOrTerminalName: string]: IProductionWithOccurrence[] + option: Option[] + alternation: Alternation[] + repetition: Repetition[] + repetitionWithSeparator: RepetitionWithSeparator[] + repetitionMandatory: RepetitionMandatory[] + repetitionMandatoryWithSeparator: RepetitionMandatoryWithSeparator[] + } = { option: [], alternation: [], repetition: [], diff --git a/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts b/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts index dcf72d6fe..eceba924c 100644 --- a/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts +++ b/packages/chevrotain/src/parse/grammar/gast/gast_resolver_public.ts @@ -23,7 +23,7 @@ export function resolveGrammar(options: { errMsgProvider: defaultGrammarResolverErrorProvider }) - const topRulesTable = {} + const topRulesTable: { [ruleName: string]: Rule } = {} forEach(options.rules, (rule) => { topRulesTable[rule.name] = rule }) diff --git a/packages/chevrotain/src/parse/grammar/interpreter.ts b/packages/chevrotain/src/parse/grammar/interpreter.ts index 63805f112..6bde0dc1f 100644 --- a/packages/chevrotain/src/parse/grammar/interpreter.ts +++ b/packages/chevrotain/src/parse/grammar/interpreter.ts @@ -147,7 +147,11 @@ export interface IFirstAfterRepetition { * it never "follows" production refs */ export class AbstractNextTerminalAfterProductionWalker extends RestWalker { - protected result = { + protected result: { + token: TokenType | undefined + occurrence: number | undefined + isEndOfRule: boolean | undefined + } = { token: undefined, occurrence: undefined, isEndOfRule: undefined @@ -250,11 +254,11 @@ export interface PartialPathAndSuffixes { export function possiblePathsFrom( targetDef: IProduction[], maxLength: number, - currPath = [] + currPath: TokenType[] = [] ): PartialPathAndSuffixes[] { // avoid side effects currPath = cloneArr(currPath) - let result = [] + let result: PartialPathAndSuffixes[] = [] let i = 0 // TODO: avoid inner funcs diff --git a/packages/chevrotain/src/parse/grammar/lookahead.ts b/packages/chevrotain/src/parse/grammar/lookahead.ts index cbdcbab44..f7634bec8 100644 --- a/packages/chevrotain/src/parse/grammar/lookahead.ts +++ b/packages/chevrotain/src/parse/grammar/lookahead.ts @@ -9,7 +9,7 @@ import { } from "@chevrotain/utils" import { possiblePathsFrom } from "./interpreter" import { RestWalker } from "./rest" -import { Predicate, TokenMatcher, lookAheadSequence } from "../parser/parser" +import { Predicate, TokenMatcher, LookAheadSequence } from "../parser/parser" import { tokenStructuredMatcher, tokenStructuredMatcherNoCategories @@ -105,7 +105,11 @@ export function buildLookaheadFuncForOptionalProd( k: number, dynamicTokensEnabled: boolean, prodType: PROD_TYPE, - lookaheadBuilder: (lookAheadSequence, TokenMatcher, boolean) => () => boolean + lookaheadBuilder: ( + lookAheadSequence: LookAheadSequence, + tokenMatcher: TokenMatcher, + dynamicTokensEnabled: boolean + ) => () => boolean ): () => boolean { const lookAheadPaths = getLookaheadPathsForOptionalProd( occurrence, @@ -124,7 +128,7 @@ export function buildLookaheadFuncForOptionalProd( export type Alternative = TokenType[][] export function buildAlternativesLookAheadFunc( - alts: lookAheadSequence[], + alts: LookAheadSequence[], hasPredicates: boolean, tokenMatcher: TokenMatcher, dynamicTokensEnabled: boolean @@ -244,7 +248,7 @@ export function buildAlternativesLookAheadFunc( } export function buildSingleAlternativeLookaheadFunction( - alt: lookAheadSequence, + alt: LookAheadSequence, tokenMatcher: TokenMatcher, dynamicTokensEnabled: boolean ): () => boolean { @@ -471,7 +475,7 @@ class InsideDefinitionFinderVisitor extends GAstVisitor { } } -function initializeArrayOfArrays(size): any[][] { +function initializeArrayOfArrays(size: number): any[][] { const result = new Array(size) for (let i = 0; i < size; i++) { result[i] = [] @@ -534,13 +538,13 @@ function isUniquePrefixHash( export function lookAheadSequenceFromAlternatives( altsDefs: IProduction[], k: number -): lookAheadSequence[] { +): LookAheadSequence[] { const partialAlts = map(altsDefs, (currAlt) => possiblePathsFrom([currAlt], 1) ) const finalResult = initializeArrayOfArrays(partialAlts.length) const altsHashes = map(partialAlts, (currAltPaths) => { - const dict = {} + const dict: { [key: string]: boolean } = {} forEach(currAltPaths, (item) => { const keys = pathToHashKeys(item.partialPath) forEach(keys, (currKey) => { @@ -611,7 +615,7 @@ export function getLookaheadPathsForOr( ruleGrammar: Rule, k: number, orProd?: Alternation -): lookAheadSequence[] { +): LookAheadSequence[] { const visitor = new InsideDefinitionFinderVisitor( occurrence, PROD_TYPE.ALTERNATION, @@ -626,7 +630,7 @@ export function getLookaheadPathsForOptionalProd( ruleGrammar: Rule, prodType: PROD_TYPE, k: number -): lookAheadSequence[] { +): LookAheadSequence[] { const insideDefVisitor = new InsideDefinitionFinderVisitor( occurrence, prodType @@ -690,7 +694,7 @@ export function isStrictPrefixOfPath( } export function areTokenCategoriesNotUsed( - lookAheadPaths: lookAheadSequence[] + lookAheadPaths: LookAheadSequence[] ): boolean { return every(lookAheadPaths, (singleAltPaths) => every(singleAltPaths, (singlePath) => diff --git a/packages/chevrotain/src/parse/grammar/rest.ts b/packages/chevrotain/src/parse/grammar/rest.ts index c668b7890..bb0c52490 100644 --- a/packages/chevrotain/src/parse/grammar/rest.ts +++ b/packages/chevrotain/src/parse/grammar/rest.ts @@ -147,17 +147,18 @@ export abstract class RestWalker { } } -function restForRepetitionWithSeparator(repSepProd, currRest, prevRest) { +function restForRepetitionWithSeparator( + repSepProd: RepetitionWithSeparator, + currRest: IProduction[], + prevRest: IProduction[] +) { const repSepRest = [ new Option({ - definition: [new Terminal({ terminalType: repSepProd.separator })].concat( - repSepProd.definition - ) - }) + definition: [ + new Terminal({ terminalType: repSepProd.separator }) as IProduction + ].concat(repSepProd.definition) + }) as IProduction ] - const fullRepSepRest: IProduction[] = repSepRest.concat( - currRest, - prevRest - ) + const fullRepSepRest: IProduction[] = repSepRest.concat(currRest, prevRest) return fullRepSepRest } diff --git a/packages/chevrotain/src/parse/parser/parser.ts b/packages/chevrotain/src/parse/parser/parser.ts index 1eeaa5d16..2d66b071c 100644 --- a/packages/chevrotain/src/parse/parser/parser.ts +++ b/packages/chevrotain/src/parse/parser/parser.ts @@ -24,7 +24,8 @@ import { IRuleConfig, IToken, TokenType, - TokenVocabulary + TokenVocabulary, + ParserMethod } from "@chevrotain/types" import { Recoverable } from "./traits/recoverable" import { LooksAhead } from "./traits/looksahead" @@ -55,7 +56,7 @@ Object.freeze(END_OF_FILE) export type TokenMatcher = (token: IToken, tokType: TokenType) => boolean -export type lookAheadSequence = TokenType[][] +export type LookAheadSequence = TokenType[][] export const DEFAULT_PARSER_CONFIG: IParserConfig = Object.freeze({ recoveryEnabled: false, @@ -104,7 +105,7 @@ export interface IParserEmptyAlternativeDefinitionError export interface IParserAmbiguousAlternativesDefinitionError extends IParserDefinitionError { - occurrence: number + occurrence: number | string alternatives: number[] } @@ -166,7 +167,10 @@ export class Parser { this.enableRecording() // Building the GAST forEach(this.definedRulesNames, (currRuleName) => { - const wrappedRule = this[currRuleName] + const wrappedRule = (this as any)[currRuleName] as ParserMethod< + unknown[], + unknown + > const originalGrammarAction = wrappedRule["originalGrammarAction"] let recordedRuleGast = undefined this.TRACE_INIT(`${currRuleName} Rule`, () => { @@ -182,7 +186,7 @@ export class Parser { } }) - let resolverErrors = [] + let resolverErrors: IParserDefinitionError[] = [] this.TRACE_INIT("Grammar Resolving", () => { resolverErrors = resolveGrammar({ rules: values(this.gastProductionsCache) diff --git a/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts b/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts index 95f80993c..d978c9438 100644 --- a/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts +++ b/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts @@ -11,6 +11,7 @@ import { IToken, ManySepMethodOpts, OrMethodOpts, + ParserMethod, SubruleMethodOpts, TokenType } from "@chevrotain/types" @@ -102,28 +103,28 @@ export class GastRecorder { */ for (let i = 0; i < 10; i++) { const idx = i > 0 ? i : "" - this[`CONSUME${idx}`] = function (arg1, arg2) { + this[`CONSUME${idx}` as "CONSUME"] = function (arg1, arg2) { return this.consumeInternalRecord(arg1, i, arg2) } - this[`SUBRULE${idx}`] = function (arg1, arg2) { - return this.subruleInternalRecord(arg1, i, arg2) + this[`SUBRULE${idx}` as "SUBRULE"] = function (arg1, arg2) { + return this.subruleInternalRecord(arg1, i, arg2) as any } - this[`OPTION${idx}`] = function (arg1) { + this[`OPTION${idx}` as "OPTION"] = function (arg1) { return this.optionInternalRecord(arg1, i) } - this[`OR${idx}`] = function (arg1) { + this[`OR${idx}` as "OR"] = function (arg1) { return this.orInternalRecord(arg1, i) } - this[`MANY${idx}`] = function (arg1) { + this[`MANY${idx}` as "MANY"] = function (arg1) { this.manyInternalRecord(i, arg1) } - this[`MANY_SEP${idx}`] = function (arg1) { + this[`MANY_SEP${idx}` as "MANY_SEP"] = function (arg1) { this.manySepFirstInternalRecord(i, arg1) } - this[`AT_LEAST_ONE${idx}`] = function (arg1) { + this[`AT_LEAST_ONE${idx}` as "AT_LEAST_ONE"] = function (arg1) { this.atLeastOneInternalRecord(i, arg1) } - this[`AT_LEAST_ONE_SEP${idx}`] = function (arg1) { + this[`AT_LEAST_ONE_SEP${idx}` as "AT_LEAST_ONE_SEP"] = function (arg1) { this.atLeastOneSepFirstInternalRecord(i, arg1) } } @@ -132,8 +133,8 @@ export class GastRecorder { this[`consume`] = function (idx, arg1, arg2) { return this.consumeInternalRecord(arg1, idx, arg2) } - this[`subrule`] = function (idx, arg1, arg2) { - return this.subruleInternalRecord(arg1, idx, arg2) + this[`subrule`] = function (idx, arg1, arg2) { + return this.subruleInternalRecord(arg1, idx, arg2) as any } this[`option`] = function (idx, arg1) { return this.optionInternalRecord(arg1, idx) @@ -163,14 +164,14 @@ export class GastRecorder { this.TRACE_INIT("Deleting Recording methods", () => { for (let i = 0; i < 10; i++) { const idx = i > 0 ? i : "" - delete this[`CONSUME${idx}`] - delete this[`SUBRULE${idx}`] - delete this[`OPTION${idx}`] - delete this[`OR${idx}`] - delete this[`MANY${idx}`] - delete this[`MANY_SEP${idx}`] - delete this[`AT_LEAST_ONE${idx}`] - delete this[`AT_LEAST_ONE_SEP${idx}`] + delete (this as any)[`CONSUME${idx}`] + delete (this as any)[`SUBRULE${idx}`] + delete (this as any)[`OPTION${idx}`] + delete (this as any)[`OR${idx}`] + delete (this as any)[`MANY${idx}`] + delete (this as any)[`MANY_SEP${idx}`] + delete (this as any)[`AT_LEAST_ONE${idx}`] + delete (this as any)[`AT_LEAST_ONE_SEP${idx}`] } delete this[`consume`] @@ -295,12 +296,12 @@ export class GastRecorder { return recordOrProd.call(this, altsOrOpts, occurrence) } - subruleInternalRecord( + subruleInternalRecord( this: MixedInParser, - ruleToCall: (idx: number) => T, + ruleToCall: ParserMethod, occurrence: number, options?: SubruleMethodOpts - ): T | CstNode { + ): R | CstNode { assertMethodIdxIsValid(occurrence) if (!ruleToCall || has(ruleToCall, "ruleName") === false) { const error: any = new Error( @@ -317,7 +318,7 @@ export class GastRecorder { } const prevProd: any = peek(this.recordingProdStack) - const ruleName = ruleToCall["ruleName"] + const ruleName = ruleToCall.ruleName const newNoneTerminal = new NonTerminal({ idx: occurrence, nonTerminalName: ruleName, @@ -431,7 +432,7 @@ function getIdxSuffix(idx: number): string { return idx === 0 ? "" : `${idx}` } -function assertMethodIdxIsValid(idx): void { +function assertMethodIdxIsValid(idx: number): void { if (idx < 0 || idx > MAX_METHOD_IDX) { const error: any = new Error( // The stack trace will contain all the needed details diff --git a/packages/chevrotain/src/parse/parser/traits/lexer_adapter.ts b/packages/chevrotain/src/parse/parser/traits/lexer_adapter.ts index 4bcbc5de8..5baf2aa07 100644 --- a/packages/chevrotain/src/parse/parser/traits/lexer_adapter.ts +++ b/packages/chevrotain/src/parse/parser/traits/lexer_adapter.ts @@ -11,7 +11,7 @@ import { MixedInParser } from "./parser_traits" */ export class LexerAdapter { tokVector: IToken[] - tokVectorLength + tokVectorLength: number currIdx: number initLexerAdapter() { diff --git a/packages/chevrotain/src/parse/parser/traits/looksahead.ts b/packages/chevrotain/src/parse/parser/traits/looksahead.ts index e11c6f706..0577ac6ee 100644 --- a/packages/chevrotain/src/parse/parser/traits/looksahead.ts +++ b/packages/chevrotain/src/parse/parser/traits/looksahead.ts @@ -8,7 +8,7 @@ import { import { forEach, has } from "@chevrotain/utils" import { DEFAULT_PARSER_CONFIG, - lookAheadSequence, + LookAheadSequence, TokenMatcher } from "../parser" import { IOrAlt, IParserConfig } from "@chevrotain/types" @@ -168,7 +168,7 @@ export class LooksAhead { lookAheadBuilderForOptional( this: MixedInParser, - alt: lookAheadSequence, + alt: LookAheadSequence, tokenMatcher: TokenMatcher, dynamicTokensEnabled: boolean ): () => boolean { @@ -181,7 +181,7 @@ export class LooksAhead { lookAheadBuilderForAlternatives( this: MixedInParser, - alts: lookAheadSequence[], + alts: LookAheadSequence[], hasPredicates: boolean, tokenMatcher: TokenMatcher, dynamicTokensEnabled: boolean diff --git a/packages/chevrotain/src/parse/parser/traits/parser_traits.ts b/packages/chevrotain/src/parse/parser/traits/parser_traits.ts index 79c9a36fc..6ea119dd9 100644 --- a/packages/chevrotain/src/parse/parser/traits/parser_traits.ts +++ b/packages/chevrotain/src/parse/parser/traits/parser_traits.ts @@ -8,7 +8,7 @@ import { TreeBuilder } from "./tree_builder" import { Parser as ParserConstructorImpel, CstParser as CstParserConstructorImpel, - EmbeddedActionsParser as EmbeddedActionsParserConstructorImpel + EmbeddedActionsParser as EmbeddedActionsParserConstructorImpl } from "../parser" import * as defs from "@chevrotain/types" import { ContentAssist } from "./context_assist" @@ -55,4 +55,4 @@ interface MixedInEmbeddedActionsParserConstructor { export const EmbeddedActionsParser: MixedInEmbeddedActionsParserConstructor = < any ->EmbeddedActionsParserConstructorImpel +>EmbeddedActionsParserConstructorImpl diff --git a/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts b/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts index 5006a3afc..8df83da23 100644 --- a/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts +++ b/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts @@ -10,6 +10,7 @@ import { IToken, ManySepMethodOpts, OrMethodOpts, + ParserMethod, SubruleMethodOpts, TokenType } from "@chevrotain/types" @@ -20,6 +21,7 @@ import { defaultGrammarValidatorErrorProvider } from "../../errors_public" import { validateRuleIsOverridden } from "../../grammar/checks" import { MixedInParser } from "./parser_traits" import { Rule, serializeGrammar } from "../../grammar/gast/gast_public" +import { IParserDefinitionError } from "../../grammar/types" /** * This trait is responsible for implementing the public API @@ -43,12 +45,12 @@ export class RecognizerApi { return this.consumeInternal(tokType, idx, options) } - subrule( + subrule( this: MixedInParser, idx: number, - ruleToCall: (idx: number) => T, + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T { + ): R { return this.subruleInternal(ruleToCall, idx, options) } @@ -164,83 +166,83 @@ export class RecognizerApi { return this.consumeInternal(tokType, 9, options) } - SUBRULE( + SUBRULE( this: MixedInParser, - ruleToCall: (idx: number) => T, + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T { + ): R { return this.subruleInternal(ruleToCall, 0, options) } - SUBRULE1( + SUBRULE1( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 1, options) } - SUBRULE2( + SUBRULE2( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 2, options) } - SUBRULE3( + SUBRULE3( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 3, options) } - SUBRULE4( + SUBRULE4( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 4, options) } - SUBRULE5( + SUBRULE5( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 5, options) } - SUBRULE6( + SUBRULE6( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 6, options) } - SUBRULE7( + SUBRULE7( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 7, options) } - SUBRULE8( + SUBRULE8( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 8, options) } - SUBRULE9( + SUBRULE9( this: MixedInParser, - ruleToCall: (idx: number) => T, - options?: SubruleMethodOpts - ): T { + ruleToCall: ParserMethod, + options?: SubruleMethodOpts + ): R { return this.subruleInternal(ruleToCall, 9, options) } @@ -658,7 +660,7 @@ export class RecognizerApi { this.definedRulesNames.push(name) const ruleImplementation = this.defineRule(name, implementation, config) - this[name] = ruleImplementation + ;(this as any)[name] = ruleImplementation return ruleImplementation } @@ -668,14 +670,15 @@ export class RecognizerApi { impl: (...implArgs: any[]) => T, config: IRuleConfig = DEFAULT_RULE_CONFIG ): (idxInCallingRule?: number, ...args: any[]) => T { - let ruleErrors = [] - ruleErrors = ruleErrors.concat( - validateRuleIsOverridden(name, this.definedRulesNames, this.className) + const ruleErrors: IParserDefinitionError[] = validateRuleIsOverridden( + name, + this.definedRulesNames, + this.className ) this.definitionErrors = this.definitionErrors.concat(ruleErrors) const ruleImplementation = this.defineRule(name, impl, config) - this[name] = ruleImplementation + ;(this as any)[name] = ruleImplementation return ruleImplementation } diff --git a/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts b/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts index e2b307119..520f323fd 100644 --- a/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts +++ b/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts @@ -10,6 +10,7 @@ import { IToken, ManySepMethodOpts, OrMethodOpts, + ParserMethod, SubruleMethodOpts, TokenType, TokenVocabulary @@ -67,7 +68,7 @@ import { Rule } from "../../grammar/gast/gast_public" * Used by the official API (recognizer_api.ts) */ export class RecognizerEngine { - isBackTrackingStack + isBackTrackingStack: boolean[] className: string RULE_STACK: number[] RULE_OCCURRENCE_STACK: number[] @@ -79,6 +80,7 @@ export class RecognizerEngine { // The shortName Index must be coded "after" the first 8bits to enable building unique lookahead keys ruleShortNameIdx: number tokenMatcher: TokenMatcher + subruleIdx: number initRecognizerEngine( tokenVocabulary: TokenVocabulary, @@ -90,6 +92,7 @@ export class RecognizerEngine { this.fullRuleNameToShort = {} this.ruleShortNameIdx = 256 this.tokenMatcher = tokenStructuredMatcherNoCategories + this.subruleIdx = 0 this.definedRulesNames = [] this.tokensMap = {} @@ -128,13 +131,13 @@ export class RecognizerEngine { } if (isArray(tokenVocabulary)) { - this.tokensMap = reduce( - tokenVocabulary, + this.tokensMap = reduce( + tokenVocabulary, (acc, tokType: TokenType) => { acc[tokType.name] = tokType return acc }, - {} + {} as { [tokenName: string]: TokenType } ) } else if ( has(tokenVocabulary, "modes") && @@ -148,7 +151,7 @@ export class RecognizerEngine { acc[tokType.name] = tokType return acc }, - {} + {} as { [tokenName: string]: TokenType } ) } else if (isObject(tokenVocabulary)) { this.tokensMap = cloneObj(tokenVocabulary) @@ -180,12 +183,12 @@ export class RecognizerEngine { augmentTokenTypes(values(this.tokensMap)) } - defineRule( + defineRule( this: MixedInParser, ruleName: string, - impl: (...implArgs: any[]) => T, - config: IRuleConfig - ): (idxInCallingRule?: number, ...args: any[]) => T { + impl: (...args: ARGS) => R, + config: IRuleConfig + ): ParserMethod { if (this.selfAnalysisDone) { throw Error( `Grammar rule <${ruleName}> may not be defined after the 'performSelfAnalysis' method has been called'\n` + @@ -210,35 +213,29 @@ export class RecognizerEngine { this.shortRuleNameToFull[shortName] = ruleName this.fullRuleNameToShort[ruleName] = shortName - function invokeRuleWithTry(this: MixedInParser, args: any[]) { + function invokeRuleWithTry(this: MixedInParser, ...args: ARGS): R { try { + this.ruleInvocationStateUpdate(shortName, ruleName, this.subruleIdx) if (this.outputCst === true) { impl.apply(this, args) const cst = this.CST_STACK[this.CST_STACK.length - 1] this.cstPostRule(cst) - return cst + return cst as unknown as R } else { return impl.apply(this, args) } } catch (e) { - return this.invokeRuleCatch(e, resyncEnabled, recoveryValueFunc) + return this.invokeRuleCatch(e, resyncEnabled, recoveryValueFunc) as R } finally { this.ruleFinallyStateUpdate() } } - const wrappedGrammarRule = function ( - this: MixedInParser, - idxInCallingRule: number = 0, - args: any[] - ) { - this.ruleInvocationStateUpdate(shortName, ruleName, idxInCallingRule) - return invokeRuleWithTry.call(this, args) - } + const wrappedGrammarRule: ParserMethod = Object.assign( + invokeRuleWithTry as any, + { ruleName, originalGrammarAction: impl } + ) - const ruleNamePropName = "ruleName" - wrappedGrammarRule[ruleNamePropName] = ruleName - wrappedGrammarRule["originalGrammarAction"] = impl return wrappedGrammarRule } @@ -247,7 +244,7 @@ export class RecognizerEngine { e: Error, resyncEnabledConfig: boolean, recoveryValueFunc: Function - ): void { + ): unknown { const isFirstInvokedRule = this.RULE_STACK.length === 1 // note the reSync is always enabled for the first rule invocation, because we must always be able to // reSync with EOF and just output some INVALID ParseTree @@ -313,11 +310,11 @@ export class RecognizerEngine { key: number ): OUT { let lookAheadFunc = this.getLaFuncFromCache(key) - let action - let predicate - if ((>actionORMethodDef).DEF !== undefined) { - action = (>actionORMethodDef).DEF - predicate = (>actionORMethodDef).GATE + let action: GrammarAction + let predicate: (this: MixedInParser) => boolean + if (typeof actionORMethodDef !== "function") { + action = actionORMethodDef.DEF + predicate = actionORMethodDef.GATE // predicate present if (predicate !== undefined) { const orgLookaheadFunction = lookAheadFunc @@ -360,10 +357,10 @@ export class RecognizerEngine { let lookAheadFunc = this.getLaFuncFromCache(key) let action - let predicate - if ((>actionORMethodDef).DEF !== undefined) { - action = (>actionORMethodDef).DEF - predicate = (>actionORMethodDef).GATE + let predicate: (this: MixedInParser) => boolean + if (typeof actionORMethodDef !== "function") { + action = actionORMethodDef.DEF + predicate = actionORMethodDef.GATE // predicate present if (predicate !== undefined) { const orgLookaheadFunction = lookAheadFunc @@ -490,10 +487,10 @@ export class RecognizerEngine { let lookaheadFunction = this.getLaFuncFromCache(key) let action - let predicate - if ((>actionORMethodDef).DEF !== undefined) { - action = (>actionORMethodDef).DEF - predicate = (>actionORMethodDef).GATE + let predicate: (this: MixedInParser) => boolean + if (typeof actionORMethodDef !== "function") { + action = actionORMethodDef.DEF + predicate = actionORMethodDef.GATE // predicate present if (predicate !== undefined) { const orgLookaheadFunction = lookaheadFunction @@ -633,9 +630,7 @@ export class RecognizerEngine { occurrence: number ): T { const laKey = this.getKeyForAutomaticLookahead(OR_IDX, occurrence) - const alts = isArray(altsOrOpts) - ? (altsOrOpts as IOrAlt[]) - : (altsOrOpts as OrMethodOpts).DEF + const alts = isArray(altsOrOpts) ? altsOrOpts : altsOrOpts.DEF const laFunc = this.getLaFuncFromCache(laKey) const altIdxToTake = laFunc.call(this, alts) @@ -666,25 +661,26 @@ export class RecognizerEngine { } } - subruleInternal( + subruleInternal( this: MixedInParser, - ruleToCall: (idx: number) => T, + ruleToCall: ParserMethod, idx: number, - options?: SubruleMethodOpts - ) { + options?: SubruleMethodOpts + ): R { let ruleResult try { const args = options !== undefined ? options.ARGS : undefined - ruleResult = ruleToCall.call(this, idx, args) + this.subruleIdx = idx + ruleResult = ruleToCall.apply(this, args) this.cstPostNonTerminal( ruleResult, options !== undefined && options.LABEL !== undefined ? options.LABEL - : (ruleToCall).ruleName + : ruleToCall.ruleName ) return ruleResult } catch (e) { - this.subruleInternalError(e, options, (ruleToCall).ruleName) + this.subruleInternalError(e, options, ruleToCall.ruleName) } } diff --git a/packages/chevrotain/src/parse/parser/traits/recoverable.ts b/packages/chevrotain/src/parse/parser/traits/recoverable.ts index 1724379e3..6fd70d26c 100644 --- a/packages/chevrotain/src/parse/parser/traits/recoverable.ts +++ b/packages/chevrotain/src/parse/parser/traits/recoverable.ts @@ -34,13 +34,13 @@ export interface IFollowKey { export const IN_RULE_RECOVERY_EXCEPTION = "InRuleRecoveryException" -export function InRuleRecoveryException(message: string) { - this.name = IN_RULE_RECOVERY_EXCEPTION - this.message = message +export class InRuleRecoveryException extends Error { + constructor(message: string) { + super(message) + this.name = IN_RULE_RECOVERY_EXCEPTION + } } -InRuleRecoveryException.prototype = Error.prototype - /** * This trait is responsible for the error recovery and fault tolerant logic */ @@ -94,7 +94,7 @@ export class Recoverable { // TODO: can the resyncTokenType be cached? const reSyncTokType = this.findReSyncTokenType() const savedLexerState = this.exportLexerState() - const resyncedTokens = [] + const resyncedTokens: IToken[] = [] let passedResyncPoint = false const nextTokenWithoutResync = this.LA(1) @@ -354,7 +354,7 @@ export class Recoverable { } reSyncTo(this: MixedInParser, tokType: TokenType): IToken[] { - const resyncedTokens = [] + const resyncedTokens: IToken[] = [] let nextTok = this.LA(1) while (this.tokenMatcher(nextTok, tokType) === false) { nextTok = this.SKIP_TOKEN() diff --git a/packages/chevrotain/src/scan/lexer_public.ts b/packages/chevrotain/src/scan/lexer_public.ts index ab5806d00..fbeab6408 100644 --- a/packages/chevrotain/src/scan/lexer_public.ts +++ b/packages/chevrotain/src/scan/lexer_public.ts @@ -33,6 +33,7 @@ import { import { augmentTokenTypes } from "./tokens" import { CustomPatternMatcherFunc, + CustomPatternMatcherReturn, ILexerConfig, ILexerDefinitionError, ILexingError, @@ -225,7 +226,7 @@ export class Lexer { const allModeNames = keys(actualDefinition.modes) - forEach( + forEach( actualDefinition.modes, (currModDef: TokenType[], currModName) => { this.TRACE_INIT(`Mode: <${currModName}> processing`, () => { @@ -234,7 +235,7 @@ export class Lexer { if (this.config.skipValidations === false) { this.TRACE_INIT(`validatePatterns`, () => { this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat( - validatePatterns(currModDef, allModeNames) + validatePatterns(currModDef, allModeNames) ) }) } @@ -406,7 +407,7 @@ export class Lexer { k, matchAltImage, longerAlt, - matchedImage, + matchedImage: string | null, payload, altPayload, imageLength, @@ -545,7 +546,7 @@ export class Lexer { if (singleCharCode !== false) { if (nextCharCode === singleCharCode) { // single character string - matchedImage = currPattern + matchedImage = currPattern as string } } else if (currConfig.isCustom === true) { match = (currPattern as IRegExpExec).exec( @@ -556,8 +557,8 @@ export class Lexer { ) if (match !== null) { matchedImage = match[0] - if (match.payload !== undefined) { - payload = match.payload + if ((match as CustomPatternMatcherReturn).payload !== undefined) { + payload = (match as CustomPatternMatcherReturn).payload } } else { matchedImage = null @@ -591,8 +592,10 @@ export class Lexer { ) if (match !== null) { matchAltImage = match[0] - if (match.payload !== undefined) { - altPayload = match.payload + if ( + (match as CustomPatternMatcherReturn).payload !== undefined + ) { + altPayload = (match as CustomPatternMatcherReturn).payload } } else { matchAltImage = null @@ -791,7 +794,7 @@ export class Lexer { // TODO: decrease this under 600 characters? inspect stripping comments option in TSC compiler private updateTokenEndLineColumnLocation( newToken: IToken, - group: string | undefined, + group: string | false, lastLTIdx: number, numOfLTsInMatch: number, line: number, @@ -919,7 +922,7 @@ export class Lexer { pattern: RegExp, text: string, offset: number - ) => string | null | RegExpExecArray + ) => string | null private matchWithTest( pattern: RegExp, @@ -933,12 +936,9 @@ export class Lexer { return null } - private matchWithExec( - pattern: RegExp, - text: string - ): string | null | RegExpExecArray { + private matchWithExec(pattern: RegExp, text: string): string | null { const regExpArray = pattern.exec(text) - return regExpArray !== null ? regExpArray[0] : regExpArray + return regExpArray !== null ? regExpArray[0] : null } // Duplicated from the parser's perf trace trait to allow future extraction diff --git a/packages/chevrotain/src/scan/reg_exp.ts b/packages/chevrotain/src/scan/reg_exp.ts index 2261a5485..2b5a7df32 100644 --- a/packages/chevrotain/src/scan/reg_exp.ts +++ b/packages/chevrotain/src/scan/reg_exp.ts @@ -1,17 +1,25 @@ -import { VERSION, BaseRegExpVisitor } from "regexp-to-ast" import { - flatten, - map, - forEach, + Alternative, + Atom, + BaseRegExpVisitor, + Character, + Disjunction, + Group, + Set, + Term, + VERSION +} from "regexp-to-ast" +import { contains, - PRINT_ERROR, - PRINT_WARNING, + every, find, + forEach, isArray, - every, + PRINT_ERROR, + PRINT_WARNING, values } from "@chevrotain/utils" -import { getRegExpAst } from "./reg_exp_parser" +import { ASTNode, getRegExpAst } from "./reg_exp_parser" import { charCodeToOptimizedIndex, minOptimizationVal } from "./lexer" const complementErrorMessage = @@ -65,7 +73,11 @@ export function getOptimizedStartCodesIndices( return [] } -export function firstCharOptimizedIndices(ast, result, ignoreCase): number[] { +export function firstCharOptimizedIndices( + ast: ASTNode, + result: { [charCode: number]: number }, + ignoreCase: boolean +): number[] { switch (ast.type) { case "Disjunction": for (let i = 0; i < ast.value.length; i++) { @@ -184,7 +196,7 @@ export function firstCharOptimizedIndices(ast, result, ignoreCase): number[] { function addOptimizedIdxToResult( code: number, - result: number[], + result: { [charCode: number]: number }, ignoreCase: boolean ) { const optimizedCharIdx = charCodeToOptimizedIndex(code) @@ -195,7 +207,10 @@ function addOptimizedIdxToResult( } } -function handleIgnoreCase(code: number, result: number[]) { +function handleIgnoreCase( + code: number, + result: { [charCode: number]: number } +) { const char = String.fromCharCode(code) const upperChar = char.toUpperCase() /* istanbul ignore else */ @@ -211,7 +226,7 @@ function handleIgnoreCase(code: number, result: number[]) { } } -function findCode(setNode, targetCharCodes) { +function findCode(setNode: Set, targetCharCodes: number[]) { return find(setNode.value, (codeOrRange) => { if (typeof codeOrRange === "number") { return contains(targetCharCodes, codeOrRange) @@ -228,8 +243,8 @@ function findCode(setNode, targetCharCodes) { }) } -function isWholeOptional(ast) { - if (ast.quantifier && ast.quantifier.atLeast === 0) { +function isWholeOptional(ast: any): boolean { + if ((ast as Atom).quantifier && (ast as Atom).quantifier.atLeast === 0) { return true } @@ -249,7 +264,7 @@ class CharCodeFinder extends BaseRegExpVisitor { super() } - visitChildren(node) { + visitChildren(node: ASTNode) { // No need to keep looking... if (this.found === true) { return @@ -269,13 +284,13 @@ class CharCodeFinder extends BaseRegExpVisitor { super.visitChildren(node) } - visitCharacter(node) { + visitCharacter(node: Character) { if (contains(this.targetCharCodes, node.value)) { this.found = true } } - visitSet(node) { + visitSet(node: Set) { if (node.complement) { if (findCode(node, this.targetCharCodes) === undefined) { this.found = true diff --git a/packages/chevrotain/src/scan/reg_exp_parser.ts b/packages/chevrotain/src/scan/reg_exp_parser.ts index 32b4cfb4e..829fc5d28 100644 --- a/packages/chevrotain/src/scan/reg_exp_parser.ts +++ b/packages/chevrotain/src/scan/reg_exp_parser.ts @@ -1,8 +1,23 @@ -import { RegExpParser, RegExpPattern } from "regexp-to-ast" +import { + Alternative, + Assertion, + Atom, + Disjunction, + RegExpParser, + RegExpPattern +} from "regexp-to-ast" -let regExpAstCache = {} +let regExpAstCache: { [regex: string]: RegExpPattern } = {} const regExpParser = new RegExpParser() +// this should be moved to regexp-to-ast +export type ASTNode = + | RegExpPattern + | Disjunction + | Alternative + | Assertion + | Atom + export function getRegExpAst(regExp: RegExp): RegExpPattern { const regExpStr = regExp.toString() if (regExpAstCache.hasOwnProperty(regExpStr)) { diff --git a/packages/chevrotain/src/scan/tokens.ts b/packages/chevrotain/src/scan/tokens.ts index 37b94e1be..345d042b6 100644 --- a/packages/chevrotain/src/scan/tokens.ts +++ b/packages/chevrotain/src/scan/tokens.ts @@ -10,9 +10,12 @@ import { isEmpty, map } from "@chevrotain/utils" -import { TokenType } from "@chevrotain/types" +import { IToken, TokenType } from "@chevrotain/types" -export function tokenStructuredMatcher(tokInstance, tokConstructor) { +export function tokenStructuredMatcher( + tokInstance: IToken, + tokConstructor: TokenType +) { const instanceType = tokInstance.tokenTypeIdx if (instanceType === tokConstructor.tokenTypeIdx) { return true @@ -26,12 +29,15 @@ export function tokenStructuredMatcher(tokInstance, tokConstructor) { // Optimized tokenMatcher in case our grammar does not use token categories // Being so tiny it is much more likely to be in-lined and this avoid the function call overhead -export function tokenStructuredMatcherNoCategories(token, tokType) { +export function tokenStructuredMatcherNoCategories( + token: IToken, + tokType: TokenType +) { return token.tokenTypeIdx === tokType.tokenTypeIdx } export let tokenShortNameIdx = 1 -export const tokenIdxToClass = {} +export const tokenIdxToClass: { [tokenIdx: number]: TokenType } = {} export function augmentTokenTypes(tokenTypes: TokenType[]): void { // collect the parent Token Types as well. diff --git a/packages/chevrotain/test/full_flow/ecma_quirks/ecma_quirks.ts b/packages/chevrotain/test/full_flow/ecma_quirks/ecma_quirks.ts index c1417eaff..c872bde49 100644 --- a/packages/chevrotain/test/full_flow/ecma_quirks/ecma_quirks.ts +++ b/packages/chevrotain/test/full_flow/ecma_quirks/ecma_quirks.ts @@ -4,7 +4,7 @@ import { EmbeddedActionsParser } from "../../../src/parse/parser/traits/parser_t import { END_OF_FILE, - lookAheadSequence, + LookAheadSequence, TokenMatcher } from "../../../src/parse/parser/parser" import { MismatchedTokenException } from "../../../src/parse/exceptions_public" @@ -96,8 +96,8 @@ class EcmaScriptQuirksParser extends EmbeddedActionsParser { ]) }) - private orgText - private textIdx + private orgText: string + private textIdx: number // lexer related methods public set textInput(newInput: string) { @@ -193,7 +193,7 @@ class EcmaScriptQuirksParser extends EmbeddedActionsParser { } lookAheadBuilderForOptional( - alt: lookAheadSequence, + alt: LookAheadSequence, tokenMatcher: TokenMatcher, dynamicTokensEnabled: boolean ): () => boolean { @@ -225,7 +225,7 @@ class EcmaScriptQuirksParser extends EmbeddedActionsParser { } lookAheadBuilderForAlternatives( - alts: lookAheadSequence[], + alts: LookAheadSequence[], hasPredicates: boolean, tokenMatcher: TokenMatcher, dynamicTokensEnabled: boolean @@ -269,7 +269,7 @@ class EcmaScriptQuirksParser extends EmbeddedActionsParser { // reuse the same parser instance. const parser = new EcmaScriptQuirksParser() -export function parse(text): any { +export function parse(text: string): any { parser.textInput = text const value = parser.statement() diff --git a/packages/chevrotain/test/full_flow/error_recovery/sql_statements/sql_recovery_parser.ts b/packages/chevrotain/test/full_flow/error_recovery/sql_statements/sql_recovery_parser.ts index 4fe4ce8c6..42ca7fe4a 100644 --- a/packages/chevrotain/test/full_flow/error_recovery/sql_statements/sql_recovery_parser.ts +++ b/packages/chevrotain/test/full_flow/error_recovery/sql_statements/sql_recovery_parser.ts @@ -91,7 +91,7 @@ export class DDLExampleRecoveryParser extends EmbeddedActionsParser { // DOCS: note how all the parsing rules in this example return a ParseTree, we require some output from the parser // to demonstrate the error recovery mechanisms. otherwise it is harder to prove we have indeed recovered. private parseDdl(): ParseTree { - const stmts = [] + const stmts: ParseTree[] = [] this.MANY(() => { this.OR([ @@ -167,7 +167,7 @@ export class DDLExampleRecoveryParser extends EmbeddedActionsParser { } private parseQualifiedName(): ParseTree { - const dots = [] + const dots: IToken[] = [] const idents = [] // parse @@ -189,8 +189,8 @@ export class DDLExampleRecoveryParser extends EmbeddedActionsParser { } private parseRecordValue(): ParseTree { - const values = [] - const commas = [] + const values: ParseTree[] = [] + const commas: IToken[] = [] // parse this.CONSUME1(LParenTok) diff --git a/packages/chevrotain/test/parse/cst_spec.ts b/packages/chevrotain/test/parse/cst_spec.ts index 3eb9d4435..69de353aa 100644 --- a/packages/chevrotain/test/parse/cst_spec.ts +++ b/packages/chevrotain/test/parse/cst_spec.ts @@ -1,9 +1,9 @@ import { createToken } from "../../src/scan/tokens_public" import { CstParser } from "../../src/parse/parser/traits/parser_traits" -import { tokenStructuredMatcher } from "../../src/scan/tokens" +import { tokenStructuredMatcher as tokenStructuredMatcherStrict } from "../../src/scan/tokens" import { createRegularToken } from "../utils/matchers" import { map } from "@chevrotain/utils" -import { CstNode, IToken, TokenType } from "@chevrotain/types" +import { CstElement, CstNode, IToken, TokenType } from "@chevrotain/types" import { expect } from "chai" function createTokenVector(tokTypes: TokenType[]): any[] { @@ -12,6 +12,11 @@ function createTokenVector(tokTypes: TokenType[]): any[] { }) } +const tokenStructuredMatcher = tokenStructuredMatcherStrict as ( + a: CstElement, + b: TokenType +) => boolean + function defineTestSuite(recoveryMode: boolean) { context(`CST Recovery: ${recoveryMode}`, () => { const A = createToken({ name: "A" }) diff --git a/packages/chevrotain/test/parse/cst_visitor_spec.ts b/packages/chevrotain/test/parse/cst_visitor_spec.ts index 7cc020306..80c1cdb88 100644 --- a/packages/chevrotain/test/parse/cst_visitor_spec.ts +++ b/packages/chevrotain/test/parse/cst_visitor_spec.ts @@ -2,7 +2,7 @@ import { createToken } from "../../src/scan/tokens_public" import { CstParser } from "../../src/parse/parser/traits/parser_traits" import { createRegularToken } from "../utils/matchers" import { keys } from "@chevrotain/utils" -import { IToken } from "@chevrotain/types" +import { CstChildrenDictionary, CstNode, IToken } from "@chevrotain/types" import { expect } from "chai" describe("The CSTVisitor", () => { @@ -56,12 +56,12 @@ describe("The CSTVisitor", () => { this.validateVisitor() } - testRule(ctx) { + testRule(ctx: any) { expect(keys(ctx)).to.deep.equal(["A", "B", "bamba"]) return this.visit(ctx.bamba[0]) } - bamba(ctx) { + bamba(ctx: any) { expect(keys(ctx)).to.deep.equal(["C"]) return 666 } @@ -86,12 +86,12 @@ describe("The CSTVisitor", () => { this.validateVisitor() } - testRule(ctx, param) { + testRule(ctx: any, param: any) { expect(keys(ctx)).to.deep.equal(["A", "B", "bamba"]) return this.visit(ctx.bamba[0], param) } - bamba(ctx, param) { + bamba(ctx: any, param: any) { // inspecting handling of optional arguments expect(this.visit(ctx.missingKey)).to.be.undefined expect(keys(ctx)).to.deep.equal(["C"]) @@ -119,7 +119,7 @@ describe("The CSTVisitor", () => { this.validateVisitor() } - bamba(ctx) { + bamba(ctx: any) { expect(keys(ctx)).to.deep.equal(["C"]) visited = true } @@ -145,12 +145,12 @@ describe("The CSTVisitor", () => { this.validateVisitor() } - testRule(ctx, param) { + testRule(ctx: any, param: any) { expect(keys(ctx)).to.deep.equal(["A", "B", "bamba"]) return this.visit(ctx["bamba"], param) } - bamba(ctx, param) { + bamba(ctx: any, param: any) { expect(keys(ctx)).to.deep.equal(["C"]) return 666 + param } @@ -176,7 +176,7 @@ describe("The CSTVisitor", () => { this.validateVisitor() } - testRule(ctx, param) {} + testRule(ctx: any, param: any) {} // missing "bamba" method } @@ -196,11 +196,11 @@ describe("The CSTVisitor", () => { this.validateVisitor() } - testRule(ctx, param) {} + testRule(ctx: any, param: any) {} - bamba(ctx, param) {} + bamba(ctx: any, param: any) {} - oops(ctx, param) {} + oops(ctx: any, param: any) {} } expect(() => new CstVisitorValidatorRedundant()).to.throw( diff --git a/packages/chevrotain/test/parse/grammar/interperter_spec.ts b/packages/chevrotain/test/parse/grammar/interperter_spec.ts index 246790dc6..fcb14c05b 100644 --- a/packages/chevrotain/test/parse/grammar/interperter_spec.ts +++ b/packages/chevrotain/test/parse/grammar/interperter_spec.ts @@ -1695,7 +1695,7 @@ describe("issue 391 - WITH_SEP variants do not take SEP into account in lookahea ) const myParser = new Issue391Parser([]) - function testInput(input) { + function testInput(input: string) { const tokens = issue391Lexer.tokenize(input).tokens myParser.input = tokens myParser.topRule() diff --git a/packages/chevrotain/test/parse/grammar/resolver_spec.ts b/packages/chevrotain/test/parse/grammar/resolver_spec.ts index 8fa705981..2dfd254bb 100644 --- a/packages/chevrotain/test/parse/grammar/resolver_spec.ts +++ b/packages/chevrotain/test/parse/grammar/resolver_spec.ts @@ -8,7 +8,7 @@ describe("The RefResolverVisitor", () => { it("will fail when trying to resolve a ref to a grammar rule that does not exist", () => { const ref = new NonTerminal({ nonTerminalName: "missingRule" }) const topLevel = new Rule({ name: "TOP", definition: [ref] }) - const topLevelRules = {} + const topLevelRules: { [ruleName: string]: Rule } = {} topLevelRules["TOP"] = topLevel const resolver = new GastRefResolverVisitor( topLevelRules, diff --git a/packages/chevrotain/test/parse/predicate_spec.ts b/packages/chevrotain/test/parse/predicate_spec.ts index 0d33ccd43..915542e19 100644 --- a/packages/chevrotain/test/parse/predicate_spec.ts +++ b/packages/chevrotain/test/parse/predicate_spec.ts @@ -275,7 +275,7 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" this.input = input } - public topRule = this.RULE("topRule", (param) => { + public topRule = this.RULE("topRule", (param?: boolean) => { return this.OR1([ { GATE: () => param, @@ -291,13 +291,13 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" const gateOpenInputA = new PredicateWithRuleOrParser([ createRegularToken(A, "a") - ]).topRule(1, [true]) + ]).topRule(true) expect(gateOpenInputA).to.equal("a") // if the predicate function still kept a reference via a closure to the original param this will not work. const gateOpenInputB = new PredicateWithRuleOrParser([ createRegularToken(B, "b") - ]).topRule(1, [false]) + ]).topRule(false) expect(gateOpenInputB).to.equal("b") }) @@ -309,9 +309,14 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" this.input = input } - public topRule = this.RULE("topRule", (param) => { + public topRule = this.RULE("topRule", (param?: boolean) => { let result = "" - result += this.CONSUME1(B).image + this.OPTION({ + GATE: () => param, + DEF: () => { + result += this.CONSUME1(B).image + } + }) return result }) @@ -320,15 +325,15 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" const parser = new PredicateWithRuleOptionParser([ createRegularToken(B, "b") ]) - const gateOpenInputB = parser.topRule(1, [false]) - expect(gateOpenInputB).to.equal("b") + const gateOpenInputB = parser.topRule(false) + expect(gateOpenInputB).to.equal("") // // if the predicate function still kept a reference via a closure to the original param this will not work. // // because the <() => param> in the OPTION will ALWAYS return false (the original param) // let gateOpenInputA = new PredicateWithRuleOptionParser([ // createRegularToken(A, "a"), // createRegularToken(B, "b") - // ]).topRule(1, [true]) + // ]).topRule(true) // expect(gateOpenInputA).to.equal("ab") }) @@ -340,7 +345,7 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" this.input = input } - public topRule = this.RULE("topRule", (param) => { + public topRule = this.RULE("topRule", (param?: boolean) => { let result = "" this.MANY({ GATE: () => param, @@ -355,7 +360,7 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" const gateOpenInputB = new PredicateWithRuleManyParser([ createRegularToken(B, "b") - ]).topRule(1, [false]) + ]).topRule(false) expect(gateOpenInputB).to.equal("b") // if the predicate function still kept a reference via a closure to the original param this will not work. @@ -365,7 +370,7 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" createRegularToken(A, "a"), createRegularToken(A, "a"), createRegularToken(B, "b") - ]).topRule(1, [true]) + ]).topRule(true) expect(gateOpenInputA).to.equal("aaab") }) @@ -377,7 +382,7 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" this.input = input } - public topRule = this.RULE("topRule", (param) => { + public topRule = this.RULE("topRule", (param?: boolean) => { let times = 0 function gateFunc() { @@ -405,7 +410,7 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" const gateOpenInputB = new PredicateWithRuleAtLeastOneParser([ createRegularToken(A, "a"), createRegularToken(B, "b") - ]).topRule(1, [false]) + ]).topRule(false) expect(gateOpenInputB).to.equal("ab") // if the predicate function still kept a reference via a closure to the original param this will not work. @@ -415,7 +420,7 @@ describe("The chevrotain support for custom gates/predicates on DSL production:" createRegularToken(A, "a"), createRegularToken(A, "a"), createRegularToken(B, "b") - ]).topRule(1, [true]) + ]).topRule(true) expect(gateOpenInputA).to.equal("aaab") }) }) diff --git a/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts b/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts index 310a58f60..9c207bbc6 100644 --- a/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts +++ b/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts @@ -273,7 +273,7 @@ describe("lookahead Regular Tokens Mode", () => { private parseManyRule(): any { let total = "" - const separators = [] + const separators: IToken[] = [] this.MANY_SEP1({ SEP: Comma, @@ -559,7 +559,7 @@ describe("lookahead Regular Tokens Mode", () => { private parseAtLeastOneRule(): any { let total = "" - const separators = [] + const separators: IToken[] = [] this.AT_LEAST_ONE_SEP1({ SEP: Comma, @@ -1968,7 +1968,7 @@ describe("lookahead Regular Tokens Mode", () => { }) } - let manyInOrBugParser + let manyInOrBugParser: ManyInOrBugParser before(() => { manyInOrBugParser = new ManyInOrBugParser() }) diff --git a/packages/chevrotain/test/parse/recognizer_spec.ts b/packages/chevrotain/test/parse/recognizer_spec.ts index 74afccffb..5a83602a3 100644 --- a/packages/chevrotain/test/parse/recognizer_spec.ts +++ b/packages/chevrotain/test/parse/recognizer_spec.ts @@ -4,7 +4,7 @@ import { EmbeddedActionsParser, CstParser } from "../../src/parse/parser/traits/parser_traits" -import { EMPTY_ALT } from "../../src/parse/parser/parser" +import { EMPTY_ALT, TokenMatcher } from "../../src/parse/parser/parser" import { expect } from "chai" import { @@ -18,13 +18,18 @@ import { augmentTokenTypes } from "../../src/scan/tokens" import { createRegularToken, setEquality } from "../utils/matchers" -import { IMultiModeLexerDefinition, IToken, TokenType } from "@chevrotain/types" +import { + IMultiModeLexerDefinition, + IToken, + ITokenConfig, + TokenType +} from "@chevrotain/types" function defineRecognizerSpecs( - contextName, - createToken, - createTokenInstance, - tokenMatcher + contextName: string, + createToken: (c: ITokenConfig) => TokenType, + createTokenInstance: typeof createRegularToken, + tokenMatcher: TokenMatcher ) { context("Recognizer " + contextName, () => { const PlusTok = createToken({ name: "PlusTok" }) @@ -108,7 +113,7 @@ function defineRecognizerSpecs( public subRule = this.RULE( "subRule", - (numFromCaller, charFromCaller) => { + (numFromCaller?: number, charFromCaller?: string) => { this.CONSUME(PlusTok) this.ACTION(() => { // side effect @@ -120,7 +125,7 @@ function defineRecognizerSpecs( public subRule2 = this.RULE( "subRule2", - (numFromCaller, charFromCaller) => { + (numFromCaller?: number, charFromCaller?: string) => { this.CONSUME(PlusTok) this.ACTION(() => { // side effect @@ -184,7 +189,7 @@ function defineRecognizerSpecs( }) it("can match an empty alternative", () => { - const input = [] + const input = [] as IToken[] const parser = new EmptyAltParser(input) expect(parser.orRule()).to.equal("EMPTY_ALT") }) @@ -325,7 +330,7 @@ function defineRecognizerSpecs( this.parseQualifiedName ) public identifier = this.RULE("identifier", this.parseIdentifier) - public idents = [] + public idents: string[] = [] private parseQualifiedName(): string[] { this.idents = [] @@ -407,7 +412,7 @@ function defineRecognizerSpecs( ) private parseQualifiedName(): string[] { - const idents = [] + const idents = [] as string[] this.AT_LEAST_ONE_SEP({ SEP: DotTok, @@ -698,7 +703,7 @@ function defineRecognizerSpecs( describe("The BaseRecognizer", () => { it("Cannot be initialized with a token vector (pre v4.0 API) ", () => { expect( - () => new EmbeddedActionsParser([createTokenInstance(PlusTok)]) + () => new EmbeddedActionsParser([createTokenInstance(PlusTok)] as any) ).to.throw( "The Parser constructor no longer accepts a token vector as the first argument" ) @@ -719,7 +724,7 @@ function defineRecognizerSpecs( it("Can skip Grammar Validations during initialization", () => { class SkipValidationsParser extends EmbeddedActionsParser { - constructor(skipValidationsValue) { + constructor(skipValidationsValue: boolean) { super(ALL_TOKENS, { skipValidations: skipValidationsValue }) @@ -848,9 +853,9 @@ function defineRecognizerSpecs( } } - expect( - () => (new WrongOrderOfSelfAnalysisParser().input = []) - ).to.throw( + expect(() => { + new WrongOrderOfSelfAnalysisParser().input = [] + }).to.throw( `Missing invocation at the end of the Parser's constructor.` ) }) diff --git a/packages/chevrotain/test/parse/traits/perf_tracer_spec.ts b/packages/chevrotain/test/parse/traits/perf_tracer_spec.ts index c37a1bad9..b0c43a992 100644 --- a/packages/chevrotain/test/parse/traits/perf_tracer_spec.ts +++ b/packages/chevrotain/test/parse/traits/perf_tracer_spec.ts @@ -4,6 +4,7 @@ import { EmbeddedActionsParser } from "../../../src/parse/parser/traits/parser_traits" import { expect } from "chai" +import { SinonSpy } from "sinon" let skipOnBrowser = describe if (typeof window !== "undefined") { @@ -11,7 +12,7 @@ if (typeof window !== "undefined") { } skipOnBrowser("Chevrotain's Init Performance Tracing", () => { - let consoleLogSpy + let consoleLogSpy: SinonSpy beforeEach(() => { // @ts-ignore @@ -26,7 +27,7 @@ skipOnBrowser("Chevrotain's Init Performance Tracing", () => { const PlusTok = createToken({ name: "PlusTok" }) class TraceParser extends EmbeddedActionsParser { - constructor(traceInitVal) { + constructor(traceInitVal: boolean | number) { super([PlusTok], { traceInitPerf: traceInitVal }) diff --git a/packages/chevrotain/test/scan/lexer_spec.ts b/packages/chevrotain/test/scan/lexer_spec.ts index 835e8f7ce..41293865a 100644 --- a/packages/chevrotain/test/scan/lexer_spec.ts +++ b/packages/chevrotain/test/scan/lexer_spec.ts @@ -29,19 +29,26 @@ import { import { setEquality } from "../utils/matchers" import { tokenStructuredMatcher } from "../../src/scan/tokens" import { - IMultiModeLexerDefinition, + ILexerConfig, ILexerErrorMessageProvider, - IToken + IMultiModeLexerDefinition, + IToken, + ITokenConfig, + TokenType } from "@chevrotain/types" import { expect } from "chai" +import { MatchArray } from "xregexp" +import { SinonSpy } from "sinon" +import { TokenMatcher } from "../../src/parse/parser/parser" const ORG_SUPPORT_STICKY = SUPPORT_STICKY + function defineLexerSpecs( - contextName, - createToken, - tokenMatcher, + contextName: string, + createToken: (c: ITokenConfig) => TokenType, + tokenMatcher: TokenMatcher, skipValidationChecks = false, - lexerConfig + lexerConfig: ILexerConfig ) { const testFull = lexerConfig.positionTracking === "full" const testStart = lexerConfig.positionTracking === "onlyStart" || testFull @@ -284,7 +291,7 @@ function defineLexerSpecs( // TODO: not sure this API allows invalid stuff const InvalidPattern = createToken({ name: "InvalidPattern", - pattern: 666 + pattern: 666 as unknown as string }) const MissingPattern = createToken({ name: "MissingPattern", @@ -629,8 +636,8 @@ function defineLexerSpecs( }) it("can transform a pattern to one with startOfInput mark ('^') #2", () => { - const orgSource = PatternNoStart.PATTERN.source - const transPattern = addStartOfInput(PatternNoStart.PATTERN) + const orgSource = (PatternNoStart.PATTERN as RegExp).source + const transPattern = addStartOfInput(PatternNoStart.PATTERN as RegExp) expect(transPattern.source).to.equal("^(?:" + orgSource + ")") expect(/^\^/.test(transPattern.source)).to.equal(true) }) @@ -1144,12 +1151,10 @@ function defineLexerSpecs( it("Will throw an error during the creation of a Lexer if the Lexer's definition is invalid", () => { expect( - () => new Lexer([EndOfInputAnchor, If, Else]), - lexerConfig + () => new Lexer([EndOfInputAnchor, If, Else], lexerConfig) ).to.throw(/Errors detected in definition of Lexer/) expect( - () => new Lexer([EndOfInputAnchor, If, Else]), - lexerConfig + () => new Lexer([EndOfInputAnchor, If, Else], lexerConfig) ).to.throw(/EndOfInputAnchor/) }) @@ -1892,81 +1897,72 @@ function defineLexerSpecs( }) }) - context("custom pattern", () => { - function defineCustomPatternSpec(variant, customPattern) { - it(variant, () => { - let time = 1 - - function extraContextValidator(text, offset, tokens, groups) { - const result = isFunction(customPattern) - ? customPattern(text, offset) - : customPattern.exec(text, offset) - if (result !== null) { - if (time === 1) { - expect(tokens).to.be.empty - time++ - } else if (time === 2) { - expect(tokens).to.have.lengthOf(2) - expect(groups.whitespace).to.have.lengthOf(2) - time++ - } else { - throw Error("Issue with Custom Token pattern context") - } - } - - return result + it("supports custom patterns", () => { + let time = 1 + + function extraContextValidator( + text: string, + offset: number, + tokens: IToken[], + groups: { [group: string]: IToken[] } + ) { + const result = /^B/.exec(text.substring(offset)) + if (result !== null) { + if (time === 1) { + expect(tokens).to.be.empty + time++ + } else if (time === 2) { + expect(tokens).to.have.lengthOf(2) + expect(groups.whitespace).to.have.lengthOf(2) + time++ + } else { + throw Error("Issue with Custom Token pattern context") } + } - const A = createToken({ - name: "A", - pattern: "A" - }) + return result + } - const B = createToken({ - name: "B", - pattern: extraContextValidator, - line_breaks: false - }) - const WS = createToken({ - name: "WS", - pattern: { - exec: (text, offset) => /^\s+/.exec(text.substring(offset)) - }, - group: "whitespace", - line_breaks: true - }) + const A = createToken({ + name: "A", + pattern: "A" + }) - const lexerDef: any = [WS, A, B] - const myLexer = new Lexer(lexerDef, lexerConfig) - const lexResult = myLexer.tokenize("B A\n B ") - expect(lexResult.tokens).to.have.length(3) - expect(tokenMatcher(lexResult.tokens[0], B)).to.be.true - expect(tokenMatcher(lexResult.tokens[1], A)).to.be.true - expect(tokenMatcher(lexResult.tokens[2], B)).to.be.true + const B = createToken({ + name: "B", + pattern: extraContextValidator, + line_breaks: false + }) + const WS = createToken({ + name: "WS", + pattern: { + exec: (text, offset) => /^\s+/.exec(text.substring(offset)) + }, + group: "whitespace", + line_breaks: true + }) - const lastToken = lexResult.tokens[2] - expect(lastToken.startOffset).to.equal(5) + const lexerDef: any = [WS, A, B] + const myLexer = new Lexer(lexerDef, lexerConfig) + const lexResult = myLexer.tokenize("B A\n B ") + expect(lexResult.tokens).to.have.length(3) + expect(tokenMatcher(lexResult.tokens[0], B)).to.be.true + expect(tokenMatcher(lexResult.tokens[1], A)).to.be.true + expect(tokenMatcher(lexResult.tokens[2], B)).to.be.true - if (testStart) { - expect(lastToken.startLine).to.equal(2) - expect(lastToken.startColumn).to.equal(2) - } + const lastToken = lexResult.tokens[2] + expect(lastToken.startOffset).to.equal(5) - if (testFull) { - expect(lastToken.endLine).to.equal(2) - expect(lastToken.endColumn).to.equal(2) - expect(lastToken.endOffset).to.equal(5) - } - }) + if (testStart) { + expect(lastToken.startLine).to.equal(2) + expect(lastToken.startColumn).to.equal(2) } - defineCustomPatternSpec( - "With short function syntax", - (text, offset) => /^B/.exec(text.substring(offset)) - ) - defineCustomPatternSpec("verbose syntax", { - exec: (text, offset) => /^B/.exec(text.substring(offset)) - }) + if (testFull) { + expect(lastToken.endLine).to.equal(2) + expect(lastToken.endColumn).to.equal(2) + expect(lastToken.endOffset).to.equal(5) + } }) }) }) @@ -1991,7 +1987,7 @@ if (typeof window !== "undefined") { } skipOnBrowser("debugging and messages and optimizations", () => { - let consoleErrorSpy, consoleWarnSpy + let consoleErrorSpy: SinonSpy, consoleWarnSpy: SinonSpy beforeEach(function () { // @ts-ignore @@ -2106,9 +2102,9 @@ skipOnBrowser("debugging and messages and optimizations", () => { }) }) -function wrapWithCustom(baseExtendToken) { - return function (...args) { - const newToken = baseExtendToken(...args) +function wrapWithCustom(baseExtendToken: (c: ITokenConfig) => TokenType) { + return function (c: ITokenConfig) { + const newToken = baseExtendToken(c) const pattern = newToken.PATTERN if ( diff --git a/packages/chevrotain/test/scan/perf_tracer_spec.ts b/packages/chevrotain/test/scan/perf_tracer_spec.ts index e53166937..910fa4ecc 100644 --- a/packages/chevrotain/test/scan/perf_tracer_spec.ts +++ b/packages/chevrotain/test/scan/perf_tracer_spec.ts @@ -1,5 +1,6 @@ import { Lexer } from "../../src/scan/lexer_public" import { expect } from "chai" +import { SinonSpy } from "sinon" let skipOnBrowser = describe if (typeof window !== "undefined") { @@ -7,7 +8,7 @@ if (typeof window !== "undefined") { } skipOnBrowser("Chevrotain's Lexer Init Performance Tracing", () => { - let consoleLogSpy + let consoleLogSpy: SinonSpy beforeEach(() => { // @ts-ignore diff --git a/packages/chevrotain/tsconfig.json b/packages/chevrotain/tsconfig.json index 87e4b525b..f20811a1a 100644 --- a/packages/chevrotain/tsconfig.json +++ b/packages/chevrotain/tsconfig.json @@ -5,7 +5,9 @@ "outDir": "lib", "baseUrl": ".", // raises too many errors to fix at once. - "strict": false + "strict": false, + "skipLibCheck": true, + "noImplicitAny": true }, "include": ["src/**/*.ts", "test/**/*.ts", "api.d.ts"] } diff --git a/packages/types/api.d.ts b/packages/types/api.d.ts index 324fd2beb..9c84e3af5 100644 --- a/packages/types/api.d.ts +++ b/packages/types/api.d.ts @@ -2,6 +2,11 @@ export as namespace chevrotain export declare const VERSION: string +export type ParserMethod = ((...args: ARGS) => R) & { + ruleName: string + originalGrammarAction: Function +} + /** * This class does not actually exists nor is exposed at runtime. * This is just a helper to avoid duplications in the Type Definitions @@ -874,22 +879,25 @@ declare abstract class BaseParser { export declare class CstParser extends BaseParser { /** * Creates a Grammar Rule + * + * Note that any parameters of your implementation must be optional as it will + * be called without parameters during the grammar recording phase. */ - protected RULE( + protected RULE void>( name: string, - implementation: (...implArgs: any[]) => any, + implementation: F, config?: IRuleConfig - ): (idxInCallingRule?: number, ...args: any[]) => CstNode + ): ParserMethod, CstNode> /** * Overrides a Grammar Rule * See usage example in: https://github.com/chevrotain/chevrotain/blob/master/examples/parser/versioning/versioning.js */ - protected OVERRIDE_RULE( + protected OVERRIDE_RULE void>( name: string, - implementation: (...implArgs: any[]) => any, + implementation: F, config?: IRuleConfig - ): (idxInCallingRule?: number, ...args: any[]) => CstNode + ): ParserMethod, CstNode> /** * Like `SUBRULE` with the numerical suffix as a parameter, e.g: @@ -899,9 +907,9 @@ export declare class CstParser extends BaseParser { * ... * @see SUBRULE */ - protected subrule( + protected subrule( idx: number, - ruleToCall: (idx: number) => CstNode, + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -922,8 +930,8 @@ export declare class CstParser extends BaseParser { * of the sub rule invocation in its rule. * */ - protected SUBRULE( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -931,8 +939,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE1( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE1( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -940,8 +948,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE2( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE2( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -949,8 +957,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE3( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE3( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -958,8 +966,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE4( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE4( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -967,8 +975,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE5( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE5( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -976,8 +984,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE6( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE6( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -985,8 +993,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE7( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE7( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -994,8 +1002,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE8( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE8( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode @@ -1003,8 +1011,8 @@ export declare class CstParser extends BaseParser { * @see SUBRULE * @hidden */ - protected SUBRULE9( - ruleToCall: (idx: number) => CstNode, + protected SUBRULE9( + ruleToCall: ParserMethod, options?: SubruleMethodOpts ): CstNode } @@ -1018,22 +1026,25 @@ export declare class CstParser extends BaseParser { export declare class EmbeddedActionsParser extends BaseParser { /** * Creates a Grammar Rule + * + * Note that any parameters of your implementation must be optional as it will + * be called without parameters during the grammar recording phase. */ - protected RULE( + protected RULE any>( name: string, - implementation: (...implArgs: any[]) => T, - config?: IRuleConfig - ): (idxInCallingRule?: number, ...args: any[]) => T + implementation: F, + config?: IRuleConfig> + ): ParserMethod, ReturnType> /** * Overrides a Grammar Rule * See usage example in: https://github.com/chevrotain/chevrotain/blob/master/examples/parser/versioning/versioning.js */ - protected OVERRIDE_RULE( + protected OVERRIDE_RULE any>( name: string, - impl: (...implArgs: any[]) => T, - config?: IRuleConfig - ): (idxInCallingRule?: number, ...args: any[]) => T + implementation: F, + config?: IRuleConfig> + ): ParserMethod, ReturnType> /** * Like `SUBRULE` with the numerical suffix as a parameter, e.g: @@ -1043,11 +1054,11 @@ export declare class EmbeddedActionsParser extends BaseParser { * ... * @see SUBRULE */ - protected subrule( + protected subrule( idx: number, - ruleToCall: (idx: number) => T, + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * The Parsing DSL Method is used by one rule to call another. @@ -1066,91 +1077,91 @@ export declare class EmbeddedActionsParser extends BaseParser { * of the sub rule invocation in its rule. * */ - protected SUBRULE( - ruleToCall: (idx: number) => T, + protected SUBRULE( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE1( - ruleToCall: (idx: number) => T, + protected SUBRULE1( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE2( - ruleToCall: (idx: number) => T, + protected SUBRULE2( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE3( - ruleToCall: (idx: number) => T, + protected SUBRULE3( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE4( - ruleToCall: (idx: number) => T, + protected SUBRULE4( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE5( - ruleToCall: (idx: number) => T, + protected SUBRULE5( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE6( - ruleToCall: (idx: number) => T, + protected SUBRULE6( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE7( - ruleToCall: (idx: number) => T, + protected SUBRULE7( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE8( - ruleToCall: (idx: number) => T, + protected SUBRULE8( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R /** * @see SUBRULE * @hidden */ - protected SUBRULE9( - ruleToCall: (idx: number) => T, + protected SUBRULE9( + ruleToCall: ParserMethod, options?: SubruleMethodOpts - ): T + ): R } export interface ILexerDefinitionError { @@ -1833,12 +1844,12 @@ export interface ConsumeMethodOpts { LABEL?: string } -export interface SubruleMethodOpts { +export interface SubruleMethodOpts { /** * The arguments to parameterized rules, see: * https://github.com/chevrotain/chevrotain/blob/master/examples/parser/parametrized_rules/parametrized.js */ - ARGS?: any[] + ARGS?: ARGS /** * A label to be used instead of the subrule's name in the created CST. */ diff --git a/packages/utils/src/api.ts b/packages/utils/src/api.ts index adecbc1c8..33a04f497 100644 --- a/packages/utils/src/api.ts +++ b/packages/utils/src/api.ts @@ -81,9 +81,9 @@ export function forEach( collection: T[], iteratorCallback: (item: T, index: number) => void ): void -export function forEach( - collection: Record, - iteratorCallback: (value: T, key: string) => void +export function forEach( + collection: Record, + iteratorCallback: (value: T, key: K) => void ): void export function forEach(collection: any, iteratorCallback: Function): void { /* istanbul ignore else */ @@ -461,3 +461,13 @@ function getCharacterFromCodePointAt(str: string, idx: number): string { ? surrogatePairCandidate : str[idx] } + +export function flatMap(arr: U[], callback: (x: U) => R[]): R[] { + const result: R[] = [] + + for (const u of arr) { + result.push(...callback(u)) + } + + return result +}