diff --git a/packages/chevrotain/api.d.ts b/packages/chevrotain/api.d.ts index ca6a75852..98f048bcd 100644 --- a/packages/chevrotain/api.d.ts +++ b/packages/chevrotain/api.d.ts @@ -86,6 +86,69 @@ declare abstract class BaseParser { */ /* protected */ ACTION(impl: () => T): T + /** + * Like `CONSUME` with the numerical suffix as a parameter, e.g: + * consume(0, X) === CONSUME(X) + * consume(1, X) === CONSUME1(X) + * consume(2, X) === CONSUME2(X) + * ... + * @see CONSUME + */ + /* protected */ consume( + idx: number, + tokType: TokenType, + options?: ConsumeMethodOpts + ): IToken + + /** + * Like `OPTION` with the numerical suffix as a parameter, e.g: + * option(0, X) === OPTION(X) + * option(1, X) === OPTION1(X) + * option(2, X) === OPTION2(X) + * ... + * @see SUBRULE + */ + /* protected */ option( + idx: number, + actionORMethodDef: GrammarAction | DSLMethodOpts + ): OUT + + /** + * Like `OR` with the numerical suffix as a parameter, e.g: + * or(0, X) === OR(X) + * or(1, X) === OR1(X) + * or(2, X) === OR2(X) + * ... + * @see OR + */ + /* protected */ or(idx: number, altsOrOpts: IOrAlt[] | OrMethodOpts): any + + /** + * Like `MANY` with the numerical suffix as a parameter, e.g: + * many(0, X) === MANY(X) + * many(1, X) === MANY1(X) + * many(2, X) === MANY2(X) + * ... + * @see MANY + */ + /* protected */ many( + idx: number, + actionORMethodDef: GrammarAction | DSLMethodOpts + ): void + + /** + * Like `AT_LEAST_ONE` with the numerical suffix as a parameter, e.g: + * atLeastOne(0, X) === AT_LEAST_ONE(X) + * atLeastOne(1, X) === AT_LEAST_ONE1(X) + * atLeastOne(2, X) === AT_LEAST_ONE2(X) + * ... + * @see AT_LEAST_ONE + */ + /* protected */ atLeastOne( + idx: number, + actionORMethodDef: GrammarAction | DSLMethodOptsWithErr + ): void + /** * * A Parsing DSL method use to consume a single Token. @@ -879,26 +942,6 @@ export declare class Parser extends BaseParser { config?: IRuleConfig ): (idxInCallingRule?: number, ...args: any[]) => T | any - /** - * The Parsing DSL Method is used by one rule to call another. - * It is equivalent to a non-Terminal in EBNF notation. - * - * This may seem redundant as it does not actually do much. - * However using it is **mandatory** for all sub rule invocations. - * - * Calling another rule without wrapping in SUBRULE(...) - * will cause errors/mistakes in the Parser's self analysis phase, - * which will lead to errors in error recovery/automatic lookahead calculation - * and any other functionality relying on the Parser's self analysis - * output. - * - * As in CONSUME the index in the method name indicates the occurrence - * of the sub rule invocation in its rule. - * - * @param ruleToCall - The rule to invoke. - * @param options - optional properties to modify the behavior of SUBRULE. - * @returns The result of invoking ruleToCall. - */ /* protected */ SUBRULE( ruleToCall: (idx: number) => T, options?: SubruleMethodOpts @@ -1001,7 +1044,7 @@ export declare class CstParser extends BaseParser { /* protected */ static performSelfAnalysis(parserInstance: Parser): void /** - * @see Parser.RULE + * Creates a Grammar Rule */ /* protected */ RULE( name: string, @@ -1010,7 +1053,8 @@ export declare class CstParser extends BaseParser { ): (idxInCallingRule?: number, ...args: any[]) => CstNode /** - * @see Parser.RULE + * Overrides a Grammar Rule + * See usage example in: https://github.com/SAP/chevrotain/blob/master/examples/parser/versioning/versioning.js */ /* protected */ OVERRIDE_RULE( name: string, @@ -1019,7 +1063,35 @@ export declare class CstParser extends BaseParser { ): (idxInCallingRule?: number, ...args: any[]) => CstNode /** - * @see Parser.SUBRULE + * Like `SUBRULE` with the numerical suffix as a parameter, e.g: + * subrule(0, X) === SUBRULE(X) + * subrule(1, X) === SUBRULE1(X) + * subrule(2, X) === SUBRULE2(X) + * ... + * @see SUBRULE + */ + /* protected */ subrule( + idx: number, + ruleToCall: (idx: number) => CstNode, + options?: SubruleMethodOpts + ): CstNode + + /** + * The Parsing DSL Method is used by one rule to call another. + * It is equivalent to a non-Terminal in EBNF notation. + * + * This may seem redundant as it does not actually do much. + * However using it is **mandatory** for all sub rule invocations. + * + * Calling another rule without wrapping in SUBRULE(...) + * will cause errors/mistakes in the Parser's self analysis phase, + * which will lead to errors in error recovery/automatic lookahead calculation + * and any other functionality relying on the Parser's self analysis + * output. + * + * As in CONSUME the index in the method name indicates the occurrence + * of the sub rule invocation in its rule. + * */ /* protected */ SUBRULE( ruleToCall: (idx: number) => CstNode, @@ -1123,7 +1195,7 @@ export declare class EmbeddedActionsParser extends BaseParser { // TODO: remove `outputCST` from the config options in the constructor /** - * @see Parser.RULE + * Creates a Grammar Rule */ /* protected */ RULE( name: string, @@ -1132,7 +1204,8 @@ export declare class EmbeddedActionsParser extends BaseParser { ): (idxInCallingRule?: number, ...args: any[]) => T /** - * @see Parser.OVERRIDE_RULE + * Overrides a Grammar Rule + * See usage example in: https://github.com/SAP/chevrotain/blob/master/examples/parser/versioning/versioning.js */ /* protected */ OVERRIDE_RULE( name: string, @@ -1141,7 +1214,35 @@ export declare class EmbeddedActionsParser extends BaseParser { ): (idxInCallingRule?: number, ...args: any[]) => T /** - * @see BaseParser.SUBRULE + * Like `SUBRULE` with the numerical suffix as a parameter, e.g: + * subrule(0, X) === SUBRULE(X) + * subrule(1, X) === SUBRULE1(X) + * subrule(2, X) === SUBRULE2(X) + * ... + * @see SUBRULE + */ + /* protected */ subrule( + idx: number, + ruleToCall: (idx: number) => T, + options?: SubruleMethodOpts + ): T + + /** + * The Parsing DSL Method is used by one rule to call another. + * It is equivalent to a non-Terminal in EBNF notation. + * + * This may seem redundant as it does not actually do much. + * However using it is **mandatory** for all sub rule invocations. + * + * Calling another rule without wrapping in SUBRULE(...) + * will cause errors/mistakes in the Parser's self analysis phase, + * which will lead to errors in error recovery/automatic lookahead calculation + * and any other functionality relying on the Parser's self analysis + * output. + * + * As in CONSUME the index in the method name indicates the occurrence + * of the sub rule invocation in its rule. + * */ /* protected */ SUBRULE( ruleToCall: (idx: number) => T, diff --git a/packages/chevrotain/benchmark_web/parsers/options.js b/packages/chevrotain/benchmark_web/parsers/options.js index 700601d17..1f096ab18 100644 --- a/packages/chevrotain/benchmark_web/parsers/options.js +++ b/packages/chevrotain/benchmark_web/parsers/options.js @@ -1,4 +1,4 @@ window.globalOptions = { - dev: { outputCst: true, maxLookahead: 2 }, - latest: { outputCst: true, maxLookahead: 2 } + dev: { outputCst: false, maxLookahead: 2 }, + latest: { outputCst: false, maxLookahead: 2 } } diff --git a/packages/chevrotain/docs/changes/CHANGELOG.md b/packages/chevrotain/docs/changes/CHANGELOG.md index ec3244296..341f2352e 100644 --- a/packages/chevrotain/docs/changes/CHANGELOG.md +++ b/packages/chevrotain/docs/changes/CHANGELOG.md @@ -2,6 +2,7 @@ #### Minor Changes +- [Larger Max numerical suffix/idx for the DSL Methods](https://github.com/SAP/chevrotain/issues/802) - [Improve duplicate DSL methods suffix error message](https://github.com/SAP/chevrotain/issues/1020) #### Bug Fixes diff --git a/packages/chevrotain/src/parse/grammar/keys.ts b/packages/chevrotain/src/parse/grammar/keys.ts index c19431bac..bd1cfeb2a 100644 --- a/packages/chevrotain/src/parse/grammar/keys.ts +++ b/packages/chevrotain/src/parse/grammar/keys.ts @@ -1,13 +1,13 @@ // Lookahead keys are 32Bit integers in the form -// TTTTTTTT-ZZZZZZZZZZZZZZZZ-YYYY-XXXX +// TTTTTTTT-ZZZZZZZZZZZZ-YYYY-XXXXXXXX // XXXX -> Occurrence Index bitmap. -// YYYY -> DSL Method Name bitmap. +// YYYY -> DSL Method Type bitmap. // ZZZZZZZZZZZZZZZ -> Rule short Index bitmap. // TTTTTTTTT -> alternation alternative index bitmap -export const BITS_FOR_METHOD_IDX = 4 -export const BITS_FOR_OCCURRENCE_IDX = 4 -export const BITS_FOR_RULE_IDX = 24 +export const BITS_FOR_METHOD_TYPE = 4 +export const BITS_FOR_OCCURRENCE_IDX = 8 +export const BITS_FOR_RULE_IDX = 12 // TODO: validation, this means that there may at most 2^8 --> 256 alternatives for an alternation. export const BITS_FOR_ALT_IDX = 8 @@ -15,12 +15,12 @@ export const BITS_FOR_ALT_IDX = 8 // being short improves the performance when composing KEYS for maps out of these // The 5 - 8 bits (16 possible values, are reserved for the DSL method indices) /* tslint:disable */ -export const OR_IDX = 1 << BITS_FOR_METHOD_IDX -export const OPTION_IDX = 2 << BITS_FOR_METHOD_IDX -export const MANY_IDX = 3 << BITS_FOR_METHOD_IDX -export const AT_LEAST_ONE_IDX = 4 << BITS_FOR_METHOD_IDX -export const MANY_SEP_IDX = 5 << BITS_FOR_METHOD_IDX -export const AT_LEAST_ONE_SEP_IDX = 6 << BITS_FOR_METHOD_IDX +export const OR_IDX = 1 << BITS_FOR_OCCURRENCE_IDX +export const OPTION_IDX = 2 << BITS_FOR_OCCURRENCE_IDX +export const MANY_IDX = 3 << BITS_FOR_OCCURRENCE_IDX +export const AT_LEAST_ONE_IDX = 4 << BITS_FOR_OCCURRENCE_IDX +export const MANY_SEP_IDX = 5 << BITS_FOR_OCCURRENCE_IDX +export const AT_LEAST_ONE_SEP_IDX = 6 << BITS_FOR_OCCURRENCE_IDX /* tslint:enable */ // this actually returns a number, but it is always used as a string (object prop key) diff --git a/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts b/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts index 41aeda6bb..5de95a5e0 100644 --- a/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts +++ b/packages/chevrotain/src/parse/parser/traits/gast_recorder.ts @@ -40,6 +40,7 @@ import { Lexer } from "../../../scan/lexer_public" import { augmentTokenTypes, hasShortKeyProperty } from "../../../scan/tokens" import { createToken, createTokenInstance } from "../../../scan/tokens_public" import { END_OF_FILE } from "../parser" +import { BITS_FOR_OCCURRENCE_IDX } from "../../grammar/keys" type ProdWithDef = IProduction & { definition?: IProduction[] } const RECORDING_NULL_OBJECT = { @@ -48,6 +49,8 @@ const RECORDING_NULL_OBJECT = { Object.freeze(RECORDING_NULL_OBJECT) const HANDLE_SEPARATOR = true +const MAX_METHOD_IDX = Math.pow(2, BITS_FOR_OCCURRENCE_IDX) - 1 + const RFT = createToken({ name: "RECORDING_PHASE_TOKEN", pattern: Lexer.NA }) augmentTokenTypes([RFT]) const RECORDING_PHASE_TOKEN = createTokenInstance( @@ -107,7 +110,7 @@ export class GastRecorder { return this.subruleInternalRecord(arg1, i, arg2) } this[`OPTION${idx}`] = function(arg1) { - this.optionInternalRecord(arg1, i) + return this.optionInternalRecord(arg1, i) } this[`OR${idx}`] = function(arg1) { return this.orInternalRecord(arg1, i) @@ -125,6 +128,27 @@ export class GastRecorder { this.atLeastOneSepFirstInternalRecord(i, arg1) } } + + // DSL methods with the idx(suffix) as an argument + this[`consume`] = function(idx, arg1, arg2) { + return this.consumeInternalRecord(arg1, idx, arg2) + } + this[`subrule`] = function(idx, arg1, arg2) { + return this.subruleInternalRecord(arg1, idx, arg2) + } + this[`option`] = function(idx, arg1) { + return this.optionInternalRecord(arg1, idx) + } + this[`or`] = function(idx, arg1) { + return this.orInternalRecord(arg1, idx) + } + this[`many`] = function(idx, arg1) { + this.manyInternalRecord(idx, arg1) + } + this[`atLeastOne`] = function(idx, arg1) { + this.atLeastOneInternalRecord(idx, arg1) + } + this.ACTION = this.ACTION_RECORD this.BACKTRACK = this.BACKTRACK_RECORD this.LA = this.LA_RECORD @@ -149,6 +173,14 @@ export class GastRecorder { delete this[`AT_LEAST_ONE${idx}`] delete this[`AT_LEAST_ONE_SEP${idx}`] } + + delete this[`consume`] + delete this[`subrule`] + delete this[`option`] + delete this[`or`] + delete this[`many`] + delete this[`atLeastOne`] + delete this.ACTION delete this.BACKTRACK delete this.LA @@ -275,6 +307,7 @@ export class GastRecorder { occurrence: number, options?: SubruleMethodOpts ): T | CstNode { + assertMethodIdxIsValid(occurrence) if (!ruleToCall || has(ruleToCall, "ruleName") === false) { const error: any = new Error( ` argument is invalid` + @@ -310,6 +343,7 @@ export class GastRecorder { occurrence: number, options: ConsumeMethodOpts ): IToken { + assertMethodIdxIsValid(occurrence) if (!hasShortKeyProperty(tokType)) { const error: any = new Error( ` argument is invalid` + @@ -340,6 +374,7 @@ function recordProd( occurrence: number, handleSep: boolean = false ): any { + assertMethodIdxIsValid(occurrence) const prevProd: any = peek(this.recordingProdStack) const grammarAction = isFunction(mainProdArg) ? mainProdArg @@ -365,6 +400,7 @@ function recordProd( } function recordOrProd(mainProdArg: any, occurrence: number): any { + assertMethodIdxIsValid(occurrence) const prevProd: any = peek(this.recordingProdStack) // Only an array of alternatives const hasOptions = isArray(mainProdArg) === false @@ -410,3 +446,16 @@ function recordOrProd(mainProdArg: any, occurrence: number): any { function getIdxSuffix(idx: number): string { return idx === 0 ? "" : `${idx}` } + +function assertMethodIdxIsValid(idx): void { + if (idx < 0 || idx > MAX_METHOD_IDX) { + const error: any = new Error( + // The stack trace will contain all the needed details + `Invalid DSL Method idx value: <${idx}>\n\t` + + `Idx value must be a none negative value smaller than ${MAX_METHOD_IDX + + 1}` + ) + error.KNOWN_RECORDER_ERROR = true + throw error + } +} diff --git a/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts b/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts index 303e8d0b7..552366cd4 100644 --- a/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts +++ b/packages/chevrotain/src/parse/parser/traits/recognizer_api.ts @@ -5,6 +5,7 @@ import { DSLMethodOptsWithErr, GrammarAction, IAnyOrAlt, + IOrAlt, IRuleConfig, ISerializedGast, IToken, @@ -22,7 +23,7 @@ import { MixedInParser } from "./parser_traits" import { Rule, serializeGrammar } from "../../grammar/gast/gast_public" /** - * This trait is responsible for implementing the offical API + * This trait is responsible for implementing the public API * for defining Chevrotain parsers, i.e: * - CONSUME * - RULE @@ -34,6 +35,56 @@ export class RecognizerApi { return impl.call(this) } + consume( + this: MixedInParser, + idx: number, + tokType: TokenType, + options?: ConsumeMethodOpts + ): IToken { + return this.consumeInternal(tokType, idx, options) + } + + subrule( + this: MixedInParser, + idx: number, + ruleToCall: (idx: number) => T, + options?: SubruleMethodOpts + ): T { + return this.subruleInternal(ruleToCall, idx, options) + } + + option( + this: MixedInParser, + idx: number, + actionORMethodDef: GrammarAction | DSLMethodOpts + ): OUT { + return this.optionInternal(actionORMethodDef, idx) + } + + or( + this: MixedInParser, + idx: number, + altsOrOpts: IOrAlt[] | OrMethodOpts + ): any { + return this.orInternal(altsOrOpts, idx) + } + + many( + this: MixedInParser, + idx: number, + actionORMethodDef: GrammarAction | DSLMethodOpts + ): void { + return this.manyInternal(idx, actionORMethodDef) + } + + atLeastOne( + this: MixedInParser, + idx: number, + actionORMethodDef: GrammarAction | DSLMethodOptsWithErr + ): void { + return this.atLeastOneInternal(idx, actionORMethodDef) + } + CONSUME( this: MixedInParser, tokType: TokenType, diff --git a/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts b/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts index 101449170..380fc40e3 100644 --- a/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts +++ b/packages/chevrotain/src/parse/parser/traits/recognizer_engine.ts @@ -30,7 +30,7 @@ import { import { AT_LEAST_ONE_IDX, AT_LEAST_ONE_SEP_IDX, - BITS_FOR_METHOD_IDX, + BITS_FOR_METHOD_TYPE, BITS_FOR_OCCURRENCE_IDX, MANY_IDX, MANY_SEP_IDX, @@ -207,7 +207,7 @@ export class RecognizerEngine { /* tslint:disable */ let shortName = this.ruleShortNameIdx << - (BITS_FOR_METHOD_IDX + BITS_FOR_OCCURRENCE_IDX) + (BITS_FOR_METHOD_TYPE + BITS_FOR_OCCURRENCE_IDX) /* tslint:enable */ this.ruleShortNameIdx++ diff --git a/packages/chevrotain/test/parse/grammar/checks_spec.ts b/packages/chevrotain/test/parse/grammar/checks_spec.ts index 5dfe8aa63..fdef95670 100644 --- a/packages/chevrotain/test/parse/grammar/checks_spec.ts +++ b/packages/chevrotain/test/parse/grammar/checks_spec.ts @@ -1,4 +1,7 @@ -import { Parser } from "../../../src/parse/parser/traits/parser_traits" +import { + CstParser, + Parser +} from "../../../src/parse/parser/traits/parser_traits" import { EMPTY_ALT, END_OF_FILE, @@ -678,6 +681,153 @@ describe("The Recorder runtime checks full flow", () => { ) }) + context( + "will throw an error when trying to init a parser with an invalid method idx", + () => { + it("consume", () => { + class InvalidIdxParser extends Parser { + constructor(input: IToken[] = []) { + super([myToken, myOtherToken]) + this.performSelfAnalysis() + this.input = input + } + + public one = this.RULE("one", () => { + this.consume(256, myToken) + }) + } + + expect(() => new InvalidIdxParser()).to.throw( + "Invalid DSL Method idx value: <256>" + ) + expect(() => new InvalidIdxParser()).to.throw( + "Idx value must be a none negative value smaller than 256" + ) + }) + + it("subrule", () => { + class InvalidIdxParser extends CstParser { + constructor(input: IToken[] = []) { + super([myToken, myOtherToken]) + this.performSelfAnalysis() + this.input = input + } + + public one = this.RULE("one", () => { + this.subrule(-1, this.two) + }) + + public two = this.RULE("two", () => { + this.consume(1, null) + }) + } + + expect(() => new InvalidIdxParser()).to.throw( + "Invalid DSL Method idx value: <-1>" + ) + expect(() => new InvalidIdxParser()).to.throw( + "Idx value must be a none negative value smaller than 256" + ) + }) + + it("option", () => { + class InvalidIdxParser extends Parser { + constructor(input: IToken[] = []) { + super([myToken, myOtherToken]) + this.performSelfAnalysis() + this.input = input + } + + public one = this.RULE("one", () => { + this.option(666, () => { + this.consume(1, myToken) + }) + }) + } + + expect(() => new InvalidIdxParser()).to.throw( + "Invalid DSL Method idx value: <666>" + ) + expect(() => new InvalidIdxParser()).to.throw( + "Idx value must be a none negative value smaller than 256" + ) + }) + + it("many", () => { + class InvalidIdxParser extends Parser { + constructor(input: IToken[] = []) { + super([myToken, myOtherToken]) + this.performSelfAnalysis() + this.input = input + } + + public one = this.RULE("one", () => { + this.many(-333, () => { + this.consume(1, myToken) + }) + }) + } + + expect(() => new InvalidIdxParser()).to.throw( + "Invalid DSL Method idx value: <-333>" + ) + expect(() => new InvalidIdxParser()).to.throw( + "Idx value must be a none negative value smaller than 256" + ) + }) + + it("atLeastOne", () => { + class InvalidIdxParser extends Parser { + constructor(input: IToken[] = []) { + super([myToken, myOtherToken]) + this.performSelfAnalysis() + this.input = input + } + + public one = this.RULE("one", () => { + this.atLeastOne(1999, () => { + this.consume(1, myToken) + }) + }) + } + + expect(() => new InvalidIdxParser()).to.throw( + "Invalid DSL Method idx value: <1999>" + ) + expect(() => new InvalidIdxParser()).to.throw( + "Idx value must be a none negative value smaller than 256" + ) + }) + + it("or", () => { + class InvalidIdxParser extends Parser { + constructor(input: IToken[] = []) { + super([myToken, myOtherToken]) + this.performSelfAnalysis() + this.input = input + } + + public one = this.RULE("one", () => { + this.or(543, [ + { + ALT: () => { + this.consume(1, myToken) + } + } + ]) + }) + } + + expect(() => new InvalidIdxParser()).to.throw( + "Invalid DSL Method idx value: <543>" + ) + expect(() => new InvalidIdxParser()).to.throw( + "Idx value must be a none negative value smaller than 256" + ) + }) + } + ) + context("augmenting error messages", () => { it("will add additional details to other runtime exceptions encountered during recording phase", () => { class OtherRecordingErrorParser extends Parser { diff --git a/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts b/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts index 9884a8847..062c4abf2 100644 --- a/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts +++ b/packages/chevrotain/test/parse/recognizer_lookahead_spec.ts @@ -14,6 +14,7 @@ describe("lookahead Regular Tokens Mode", () => { let SevenTok = createToken({ name: "SevenTok" }) let EightTok = createToken({ name: "EightTok" }) let NineTok = createToken({ name: "NineTok" }) + let TenTok = createToken({ name: "TenTok" }) let Comma = createToken({ name: "Comma" }) const ALL_TOKENS = [ @@ -81,6 +82,11 @@ describe("lookahead Regular Tokens Mode", () => { total += "5" }) + this.option(20, () => { + this.CONSUME1(SixTok) + total += "6" + }) + return total } } @@ -114,6 +120,12 @@ describe("lookahead Regular Tokens Mode", () => { let parser = new OptionsImplicitLookAheadParser(input) expect(parser.manyOptionsRule()).to.equal("5") }) + + it("can automatically compute lookahead for option(idx, ...)", () => { + let input = [createRegularToken(SixTok)] + let parser = new OptionsImplicitLookAheadParser(input) + expect(parser.manyOptionsRule()).to.equal("6") + }) }) describe("The implicit lookahead calculation functionality of the Recognizer For MANY", () => { @@ -185,6 +197,11 @@ describe("lookahead Regular Tokens Mode", () => { total += "9" }) + this.many(20, () => { + this.CONSUME1(TenTok) + total += "10" + }) + return total } } @@ -243,6 +260,12 @@ describe("lookahead Regular Tokens Mode", () => { expect(parser.manyRule()).to.equal("9") }) + it("can automatically compute lookahead for many(idx, ...)", () => { + let input = [createRegularToken(TenTok)] + let parser = new ManyImplicitLookAheadParser(input) + expect(parser.manyRule()).to.equal("10") + }) + it("can accept lookahead function param for flow mixing several MANYs", () => { let input = [ createRegularToken(OneTok), @@ -510,6 +533,11 @@ describe("lookahead Regular Tokens Mode", () => { total += "9" }) + this.atLeastOne(32, () => { + this.CONSUME1(TenTok) + total += "10" + }) + return total } } @@ -528,10 +556,11 @@ describe("lookahead Regular Tokens Mode", () => { createRegularToken(EightTok), createRegularToken(EightTok), createRegularToken(EightTok), - createRegularToken(NineTok) + createRegularToken(NineTok), + createRegularToken(TenTok) ] let parser = new AtLeastOneImplicitLookAheadParser(input) - expect(parser.atLeastOneRule()).to.equal("1223445678889") + expect(parser.atLeastOneRule()).to.equal("122344567888910") }) it("will fail when zero occurrences of AT_LEAST_ONE in input", () => { @@ -617,7 +646,7 @@ describe("lookahead Regular Tokens Mode", () => { this.AT_LEAST_ONE_SEP5({ SEP: Comma, DEF: () => { - this.CONSUME1(FiveTok) + this.consume(72, FiveTok) total += "5" } }) @@ -889,6 +918,39 @@ describe("lookahead Regular Tokens Mode", () => { } ]) + this.or(45, [ + { + ALT: () => { + this.CONSUME6(TwoTok) + total += "F2" + } + }, + { + ALT: () => { + this.CONSUME6(OneTok) + total += "F1" + } + }, + { + ALT: () => { + this.CONSUME6(ThreeTok) + total += "F3" + } + }, + { + ALT: () => { + this.CONSUME6(FourTok) + total += "F4" + } + }, + { + ALT: () => { + this.consume(66, FiveTok) + total += "F5" + } + } + ]) + return total } } @@ -899,10 +961,11 @@ describe("lookahead Regular Tokens Mode", () => { createRegularToken(TwoTok), createRegularToken(ThreeTok), createRegularToken(FourTok), - createRegularToken(FiveTok) + createRegularToken(FiveTok), + createRegularToken(ThreeTok) ] let parser = new OrImplicitLookAheadParser(input) - expect(parser.orRule()).to.equal("A1B2C3D4E5") + expect(parser.orRule()).to.equal("A1B2C3D4E5F3") }) it("will fail when none of the alternatives match", () => { diff --git a/packages/chevrotain/test/parse/recognizer_spec.ts b/packages/chevrotain/test/parse/recognizer_spec.ts index e266697a9..77b4497f7 100644 --- a/packages/chevrotain/test/parse/recognizer_spec.ts +++ b/packages/chevrotain/test/parse/recognizer_spec.ts @@ -1,6 +1,9 @@ import { EOF, createToken } from "../../src/scan/tokens_public" import { Lexer } from "../../src/scan/lexer_public" -import { Parser } from "../../src/parse/parser/traits/parser_traits" +import { + EmbeddedActionsParser, + Parser +} from "../../src/parse/parser/traits/parser_traits" import { EMPTY_ALT } from "../../src/parse/parser/parser" import { @@ -35,7 +38,7 @@ function defineRecognizerSpecs( describe("The Parsing DSL", () => { it("provides a production SUBRULE1-5 that invokes another rule", () => { - class SubRuleTestParser extends Parser { + class SubRuleTestParser extends EmbeddedActionsParser { private result = "" private index = 1 @@ -50,7 +53,7 @@ function defineRecognizerSpecs( this.SUBRULE1(this.subRule) this.SUBRULE2(this.subRule) this.SUBRULE3(this.subRule) - this.SUBRULE4(this.subRule) + this.subrule(66, this.subRule) this.SUBRULE5(this.subRule) return this.result }) @@ -659,7 +662,7 @@ function defineRecognizerSpecs( this.input = input } - public or = this.RULE("or", () => { + public orRule = this.RULE("orRule", () => { return this.OR([ { ALT: () => { @@ -680,10 +683,10 @@ function defineRecognizerSpecs( let parser = new OrExpressionParser([]) parser.input = [createTokenInstance(MinusTok)] - expect(parser.or()).to.equal(666) + expect(parser.orRule()).to.equal(666) parser.input = [createTokenInstance(PlusTok)] - expect(parser.or()).to.equal("bamba") + expect(parser.orRule()).to.equal("bamba") }) it("OPTION will return the grammar action value or undefined if the option was not taken", () => { @@ -695,7 +698,7 @@ function defineRecognizerSpecs( this.input = input } - public option = this.RULE("option", () => { + public optionRule = this.RULE("optionRule", () => { return this.OPTION(() => { this.CONSUME(IdentTok) return "bamba" @@ -706,10 +709,10 @@ function defineRecognizerSpecs( let parser = new OptionExpressionParser([]) parser.input = [createTokenInstance(IdentTok)] - expect(parser.option()).to.equal("bamba") + expect(parser.optionRule()).to.equal("bamba") parser.input = [createTokenInstance(IntTok)] - expect(parser.option()).to.be.undefined + expect(parser.optionRule()).to.be.undefined }) })