From 72d5c3f1ae827be3f5f39fee35c8b99270353208 Mon Sep 17 00:00:00 2001 From: Clayton Carter Date: Tue, 9 Jan 2024 23:27:09 -0500 Subject: [PATCH 1/3] test: add tests for PHP WASM tree-sitter grammar These tests and the scopes are meant to mimic the TextMate grammar and scopes, but the new tree-sitter grammar deviates in a few ways. The adjustments needed to make these pass are in the next commit. --- .../language-php/spec/tree-sitter-helpers.js | 171 ++++++++++++++++++ .../spec/wasm-tree-sitter-spec.js | 166 +++++++++++++++++ 2 files changed, 337 insertions(+) create mode 100644 packages/language-php/spec/tree-sitter-helpers.js create mode 100644 packages/language-php/spec/wasm-tree-sitter-spec.js diff --git a/packages/language-php/spec/tree-sitter-helpers.js b/packages/language-php/spec/tree-sitter-helpers.js new file mode 100644 index 0000000000..4fba21fe89 --- /dev/null +++ b/packages/language-php/spec/tree-sitter-helpers.js @@ -0,0 +1,171 @@ +const dedent = require("dedent"); + +module.exports = { + // Taken from Atom source at + // https://github.com/atom/atom/blob/b3d3a52d9e4eb41f33df7b91ad1f8a2657a04487/spec/tree-sitter-language-mode-spec.js#L47-L55 + // Not used in tests, but included for reference. I recall that it works by + // tokenizing lines and then lising the scopes for each token. This allows + // specs like: + // + // editor.setPhpText(` + // $foo + 1; + // $bar->baz; + // `) + // expectTokensToEqual( + // editor, + // [ + // [ + // {text: '$foo', scopes: [...]} + // {text: '+', scopes: [...]} + // {text: '1', scopes: [...]} + // ], + // [ + // {text: '$bar', scopes: [...]} + // {text: '->', scopes: [...]} + // {text: 'baz', scopes: [...]} + // ] + // ] + // ) + expectTokensToEqual(editor, expectedTokenLines, startingRow = 1) { + const lastRow = editor.getLastScreenRow(); + + for (let row = startingRow; row <= lastRow - startingRow; row++) { + const tokenLine = editor + .tokensForScreenRow(row) + .map(({ text, scopes }) => ({ + text, + scopes: scopes.map((scope) => + scope + .split(" ") + .map((className) => className.replace("syntax--", "")) + .join(".") + ), + })); + + const expectedTokenLine = expectedTokenLines[row - startingRow]; + + expect(tokenLine.length).toEqual(expectedTokenLine.length); + for (let i = 0; i < tokenLine.length; i++) { + expect(tokenLine[i].text).toEqual( + expectedTokenLine[i].text, + `Token ${i}, row: ${row}` + ); + expect(tokenLine[i].scopes).toEqual( + expectedTokenLine[i].scopes, + `Token ${i}, row: ${row}, token: '${tokenLine[i].text}'` + ); + } + } + }, + + /** + * A matcher to compare scopes applied by a tree-sitter grammar on a character + * by character basis. + * + * @param {array} posn Buffer position to be examined. A Point in the form [row, col]. Both are 0 based. + * @param {string|array} token The token to be matched at the given position. Mostly just to make the tests easier to read. + * @param {?array} expected The scopes that should be present. + * @param {Object} options Options to change what is asserted. + */ + toHaveScopes(posn, token, expected, options = {}) { + if (token === undefined) { + throw new Error( + 'toHaveScopes must be called with at least 2 parameters' + ); + } + if (expected === undefined) { + expected = token; + token = ''; + } + + // remove base scopes by default + const removeBaseScopes = options.removeBaseScopes ?? true; + const filterBaseScopes = (scope) => + ( + removeBaseScopes && + scope !== "text.html.php" && + scope !== "source.php" + ); + + // this.actual is a Pulsar TextEditor + const line = this.actual.getBuffer().lineForRow(posn[0]); + const caret = " ".repeat(posn[1]) + "^"; + + const actualToken = this.actual + .getTextInBufferRange([posn, [posn[0], posn[1] + token.length]]); + + if (actualToken !== token) { + this.message = () => ` + Failure: Tokens did not match at position [${posn.join(", ")}]: +${line} +${caret} + Expected token: ${token} +` + return false; + } + + const actualScopes = this.actual + .scopeDescriptorForBufferPosition(posn) + .scopes + .filter(filterBaseScopes); + + const notExpected = actualScopes.filter((scope) => !expected.includes(scope)); + const notReceived = expected.filter((scope) => !actualScopes.includes(scope)); + + const pass = notExpected.length === 0 && notReceived.length === 0; + + if (pass) { + this.message = () => "Scopes matched"; + return true; + } + + this.message = () => + ` + Failure: Scopes did not match at position [${posn.join(", ")}]: +${line} +${caret} + These scopes were expected but not received: + ${notReceived.join(", ")} + These scopes were received but not expected: + ${notExpected.join(", ")} + ` + + ( + (options.showAllScopes ?? false) + ? ` + These were all scopes recieved: + ${actualScopes.join(", ")} + These were all scopes expected: + ${expected.join(", ")} + ` + : '' + ); + + return false; + }, + + /** + * Wrap a code snippet in PHP tags, insert it into an editor, and wait for the + * language mode to be ready. + * @param {string} content a PHP code snippet + * @return {Promise} resolves when the editor language mode is ready + */ + async setPhpText(content) { + this.setText(` { + const subscription = editor + .getBuffer() + .getLanguageMode() + .onDidChangeHighlighting(() => { + subscription.dispose(); + resolve(); + }); + }); + }, +}; diff --git a/packages/language-php/spec/wasm-tree-sitter-spec.js b/packages/language-php/spec/wasm-tree-sitter-spec.js new file mode 100644 index 0000000000..55ab5c5ab2 --- /dev/null +++ b/packages/language-php/spec/wasm-tree-sitter-spec.js @@ -0,0 +1,166 @@ +const {toHaveScopes, setPhpText} = require('./tree-sitter-helpers') + +describe("Tree-sitter PHP grammar", () => { + var editor; + + beforeEach(async () => { + atom.config.set("core.useTreeSitterParsers", true); + atom.config.set("core.useExperimentalModernTreeSitter", true); + await atom.packages.activatePackage("language-php"); + editor = await atom.workspace.open("foo.php"); + editor.setPhpText = setPhpText; + }); + + beforeEach(function () { + this.addMatchers({toHaveScopes}); + }); + + describe("loading the grammars", () => { + it('loads the wrapper "HTML" grammar', () => { + embeddingGrammar = atom.grammars.grammarForScopeName("text.html.php"); + expect(embeddingGrammar).toBeTruthy(); + expect(embeddingGrammar.scopeName).toBe("text.html.php"); + expect(embeddingGrammar.constructor.name).toBe("WASMTreeSitterGrammar"); + + // injections + expect(embeddingGrammar.injectionPointsByType.program).toBeTruthy(); + expect(embeddingGrammar.injectionPointsByType.comment).toBeTruthy(); + }) + }); + + describe("operators", () => { + it("scopes =", async () => { + await editor.setPhpText('$test = 1;'); + + expect(editor).toHaveScopes([1, 0], '$', ["variable.other.php", "punctuation.definition.variable.php"]); + expect(editor).toHaveScopes([1, 5], ' ', []); + expect(editor).toHaveScopes([1, 6], '=', ["keyword.operator.assignment.php"]); + expect(editor).toHaveScopes([1, 8], '1', ["constant.numeric.decimal.php"]); + expect(editor).toHaveScopes([1, 9], ';', ["punctuation.terminator.expression.php"]); + }); + + it("scopes +", async () => { + await editor.setPhpText('1 + 2;'); + + expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.php"]); + expect(editor).toHaveScopes([1, 2], '+', ["keyword.operator.arithmetic.php"]); + expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.php"]); + }); + + it("scopes %", async () => { + await editor.setPhpText('1 % 2;'); + + expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.php"]); + expect(editor).toHaveScopes([1, 2], '%', ["keyword.operator.arithmetic.php"]); + expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.php"]); + }); + + it("scopes instanceof", async () => { + await editor.setPhpText('$x instanceof Foo;'); + + expect(editor).toHaveScopes([1, 0], '$', ["variable.other.php", "punctuation.definition.variable.php"]); + expect(editor).toHaveScopes([1, 1], 'x', ["variable.other.php"]); + expect(editor).toHaveScopes([1, 3], 'instanceof', ["keyword.operator.type.php"]); + expect(editor).toHaveScopes([1, 14], 'Foo', ["support.class.php"]); + }); + + describe("combined operators", () => { + it("scopes ===", async () => { + await editor.setPhpText('$test === 2;'); + + expect(editor).toHaveScopes([1, 6], ["keyword.operator.comparison.php"]); + }); + + it("scopes +=", async () => { + await editor.setPhpText('$test += 2;'); + + expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.php"]); + }); + + it("scopes ??=", async () => { + await editor.setPhpText('$test ??= true;'); + + expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.php"]); + }); + }); + }); + + it("should tokenize $this", async () => { + await editor.setPhpText("$this;"); + + expect(editor).toHaveScopes([1, 0], '$', ["variable.language.builtin.this.php", "punctuation.definition.variable.php"]); + expect(editor).toHaveScopes([1, 1], 'this', ["variable.language.builtin.this.php"]); + + await editor.setPhpText("$thistles;"); + + expect(editor).toHaveScopes([1, 0], '$', ["variable.other.php", "punctuation.definition.variable.php"]); + expect(editor).toHaveScopes([1, 1], 'thistles', ["variable.other.php"]); + }); + + describe("use declarations", () => { + it("scopes basic use statements", async () => { + await editor.setPhpText("use Foo;"); + + expect(editor).toHaveScopes([1, 0], 'use', ["keyword.other.use.php"]); + expect(editor).toHaveScopes([1, 4], 'Foo', ["support.class.php"]); + expect(editor).toHaveScopes([1, 7], ';', ["punctuation.terminator.expression.php"]); + + await editor.setPhpText("use My\\Full\\NSname;"); + + expect(editor).toHaveScopes([1, 0], 'use', ["keyword.other.use.php"]); + expect(editor).toHaveScopes([1, 4], 'My', ["support.other.namespace.php"]); + expect(editor).toHaveScopes([1, 6], '\\', ["support.other.namespace.php", "punctuation.separator.inheritance.php"]); + expect(editor).toHaveScopes([1, 7], 'Full', ["support.other.namespace.php"]); + expect(editor).toHaveScopes([1, 11], '\\', ["support.other.namespace.php","punctuation.separator.inheritance.php"]); + expect(editor).toHaveScopes([1, 12], 'NSname', ["support.class.php"]); + expect(editor).toHaveScopes([1, 18], ';', ["punctuation.terminator.expression.php"]); + }); + }); + + describe("classes", () => { + it("scopes class declarations", async () => { + await editor.setPhpText("class Test {}"); + + expect(editor).toHaveScopes([1, 0], 'class', ["storage.type.class.php"]); + expect(editor).toHaveScopes([1, 6], 'Test', ["entity.name.type.class.php"]); + expect(editor).toHaveScopes([1, 11], '{', ['punctuation.definition.block.begin.bracket.curly.php']); + expect(editor).toHaveScopes([1, 12], '}', ['punctuation.definition.block.end.bracket.curly.php']); + }); + + it("scopes class instantiation", async () => { + await editor.setPhpText("$a = new ClassName();"); + + expect(editor).toHaveScopes([1, 5], 'new', ["keyword.other.new.php"]); + expect(editor).toHaveScopes([1, 9], 'ClassName', ["support.class.php"]); + expect(editor).toHaveScopes([1, 18], '(', ["punctuation.definition.begin.bracket.round.php"]); + expect(editor).toHaveScopes([1, 19], ')', ["punctuation.definition.end.bracket.round.php"]); + expect(editor).toHaveScopes([1, 20], ';', ["punctuation.terminator.expression.php"]); + }); + + it("scopes class modifiers", async () => { + await editor.setPhpText("abstract class Test {}"); + + expect(editor).toHaveScopes([1, 0], 'abstract', ["storage.modifier.abstract.php"]); + expect(editor).toHaveScopes([1, 9], 'class', ["storage.type.class.php"]); + + await editor.setPhpText("final class Test {}"); + + expect(editor).toHaveScopes([1, 0], 'final', ["storage.modifier.final.php"]); + expect(editor).toHaveScopes([1, 6], 'class', ["storage.type.class.php"]); + }); + }); + + describe("phpdoc", () => { + it("scopes @return tags", async () => { + await editor.setPhpText("/** @return Foo */"); + + expect(editor).toHaveScopes([1, 0], '/**', ['comment.block.documentation.phpdoc.php', 'punctuation.definition.begin.comment.phpdoc.php']); + expect(editor).toHaveScopes([1, 4], '@return', ['comment.block.documentation.phpdoc.php', 'entity.name.tag.phpdoc.php']); + expect(editor).toHaveScopes([1, 12], 'Foo', ['comment.block.documentation.phpdoc.php', 'storage.type.instance.phpdoc.php']); + expect(editor).toHaveScopes([1, 15], '<', ['comment.block.documentation.phpdoc.php']); + expect(editor).toHaveScopes([1, 16], 'Bar', ['comment.block.documentation.phpdoc.php', 'storage.type.instance.phpdoc.php']); + expect(editor).toHaveScopes([1, 19], '>', ['comment.block.documentation.phpdoc.php']); + expect(editor).toHaveScopes([1, 21], '*/', ['comment.block.documentation.phpdoc.php', 'punctuation.definition.end.comment.phpdoc.php']); + }); + }); +}); From f9a544604a8e8100098c61394e7406ffb54de803 Mon Sep 17 00:00:00 2001 From: Clayton Carter Date: Tue, 9 Jan 2024 23:34:32 -0500 Subject: [PATCH 2/3] test: make most of the tests pass This updates the tests to match the output of the new WASM grammar, except where I think the WASM grammar may be wrong. --- .../spec/wasm-tree-sitter-spec.js | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/packages/language-php/spec/wasm-tree-sitter-spec.js b/packages/language-php/spec/wasm-tree-sitter-spec.js index 55ab5c5ab2..42f582979e 100644 --- a/packages/language-php/spec/wasm-tree-sitter-spec.js +++ b/packages/language-php/spec/wasm-tree-sitter-spec.js @@ -35,24 +35,23 @@ describe("Tree-sitter PHP grammar", () => { expect(editor).toHaveScopes([1, 0], '$', ["variable.other.php", "punctuation.definition.variable.php"]); expect(editor).toHaveScopes([1, 5], ' ', []); expect(editor).toHaveScopes([1, 6], '=', ["keyword.operator.assignment.php"]); - expect(editor).toHaveScopes([1, 8], '1', ["constant.numeric.decimal.php"]); - expect(editor).toHaveScopes([1, 9], ';', ["punctuation.terminator.expression.php"]); + expect(editor).toHaveScopes([1, 8], '1', ["constant.numeric.decimal.integer.php"]); }); it("scopes +", async () => { await editor.setPhpText('1 + 2;'); - expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.php"]); + expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.integer.php"]); expect(editor).toHaveScopes([1, 2], '+', ["keyword.operator.arithmetic.php"]); - expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.php"]); + expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.integer.php"]); }); it("scopes %", async () => { await editor.setPhpText('1 % 2;'); - expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.php"]); + expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.integer.php"]); expect(editor).toHaveScopes([1, 2], '%', ["keyword.operator.arithmetic.php"]); - expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.php"]); + expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.integer.php"]); }); it("scopes instanceof", async () => { @@ -61,7 +60,7 @@ describe("Tree-sitter PHP grammar", () => { expect(editor).toHaveScopes([1, 0], '$', ["variable.other.php", "punctuation.definition.variable.php"]); expect(editor).toHaveScopes([1, 1], 'x', ["variable.other.php"]); expect(editor).toHaveScopes([1, 3], 'instanceof', ["keyword.operator.type.php"]); - expect(editor).toHaveScopes([1, 14], 'Foo', ["support.class.php"]); + expect(editor).toHaveScopes([1, 14], 'Foo', ["support.other.function.constructor.php"]); }); describe("combined operators", () => { @@ -74,18 +73,18 @@ describe("Tree-sitter PHP grammar", () => { it("scopes +=", async () => { await editor.setPhpText('$test += 2;'); - expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.php"]); + expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.compound.php"]); }); it("scopes ??=", async () => { await editor.setPhpText('$test ??= true;'); - expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.php"]); + expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.compound.php"]); }); }); }); - it("should tokenize $this", async () => { + it("scopes $this", async () => { await editor.setPhpText("$this;"); expect(editor).toHaveScopes([1, 0], '$', ["variable.language.builtin.this.php", "punctuation.definition.variable.php"]); @@ -101,19 +100,17 @@ describe("Tree-sitter PHP grammar", () => { it("scopes basic use statements", async () => { await editor.setPhpText("use Foo;"); - expect(editor).toHaveScopes([1, 0], 'use', ["keyword.other.use.php"]); - expect(editor).toHaveScopes([1, 4], 'Foo', ["support.class.php"]); - expect(editor).toHaveScopes([1, 7], ';', ["punctuation.terminator.expression.php"]); + expect(editor).toHaveScopes([1, 0], 'use', ["keyword.control.use.php"]); + expect(editor).toHaveScopes([1, 4], 'Foo', ["entity.name.type.namespace.php"]); await editor.setPhpText("use My\\Full\\NSname;"); - expect(editor).toHaveScopes([1, 0], 'use', ["keyword.other.use.php"]); - expect(editor).toHaveScopes([1, 4], 'My', ["support.other.namespace.php"]); - expect(editor).toHaveScopes([1, 6], '\\', ["support.other.namespace.php", "punctuation.separator.inheritance.php"]); - expect(editor).toHaveScopes([1, 7], 'Full', ["support.other.namespace.php"]); - expect(editor).toHaveScopes([1, 11], '\\', ["support.other.namespace.php","punctuation.separator.inheritance.php"]); - expect(editor).toHaveScopes([1, 12], 'NSname', ["support.class.php"]); - expect(editor).toHaveScopes([1, 18], ';', ["punctuation.terminator.expression.php"]); + expect(editor).toHaveScopes([1, 0], 'use', ["keyword.control.use.php"]); + expect(editor).toHaveScopes([1, 4], 'My', ["entity.name.type.namespace.php"]); + expect(editor).toHaveScopes([1, 6], '\\', ["punctuation.operator.namespace.php"]); + expect(editor).toHaveScopes([1, 7], 'Full', ["entity.name.type.namespace.php"]); + expect(editor).toHaveScopes([1, 11], '\\', ["punctuation.operator.namespace.php"]); + expect(editor).toHaveScopes([1, 12], 'NSname', ["entity.name.type.namespace.php"]); }); }); @@ -130,22 +127,21 @@ describe("Tree-sitter PHP grammar", () => { it("scopes class instantiation", async () => { await editor.setPhpText("$a = new ClassName();"); - expect(editor).toHaveScopes([1, 5], 'new', ["keyword.other.new.php"]); - expect(editor).toHaveScopes([1, 9], 'ClassName', ["support.class.php"]); + expect(editor).toHaveScopes([1, 5], 'new', ["keyword.control.new.php"]); + expect(editor).toHaveScopes([1, 9], 'ClassName', ["support.other.function.constructor.php"]); expect(editor).toHaveScopes([1, 18], '(', ["punctuation.definition.begin.bracket.round.php"]); expect(editor).toHaveScopes([1, 19], ')', ["punctuation.definition.end.bracket.round.php"]); - expect(editor).toHaveScopes([1, 20], ';', ["punctuation.terminator.expression.php"]); }); it("scopes class modifiers", async () => { await editor.setPhpText("abstract class Test {}"); - expect(editor).toHaveScopes([1, 0], 'abstract', ["storage.modifier.abstract.php"]); + expect(editor).toHaveScopes([1, 0], 'abstract', ["keyword.control.abstract.php"]); expect(editor).toHaveScopes([1, 9], 'class', ["storage.type.class.php"]); await editor.setPhpText("final class Test {}"); - expect(editor).toHaveScopes([1, 0], 'final', ["storage.modifier.final.php"]); + expect(editor).toHaveScopes([1, 0], 'final', ["keyword.control.final.php"]); expect(editor).toHaveScopes([1, 6], 'class', ["storage.type.class.php"]); }); }); From 6a7afbef4c81653faa83768dbb2d75c079b60305 Mon Sep 17 00:00:00 2001 From: Clayton Carter Date: Tue, 9 Jan 2024 23:55:53 -0500 Subject: [PATCH 3/3] test: add more failing tests These cover some of the cases mentioned at https://github.com/pulsar-edit/pulsar/pull/852#issuecomment-1884106401 --- .../spec/wasm-tree-sitter-spec.js | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/language-php/spec/wasm-tree-sitter-spec.js b/packages/language-php/spec/wasm-tree-sitter-spec.js index 42f582979e..19de42296e 100644 --- a/packages/language-php/spec/wasm-tree-sitter-spec.js +++ b/packages/language-php/spec/wasm-tree-sitter-spec.js @@ -28,6 +28,27 @@ describe("Tree-sitter PHP grammar", () => { }) }); + describe('php tags', () => { + it('scopes regular opening tags', async () => { + await editor.setPhpText(''); + + expect(editor).toHaveScopes([0, 0], ' { + editor.setText(''); + await editor.languageMode.ready; + + expect(editor).toHaveScopes([0, 0], ' { + await editor.setPhpText('?>'); + + expect(editor).toHaveScopes([1, 0], '?>', ['punctuation.section.embedded.end.php']); + }) + }); + describe("operators", () => { it("scopes =", async () => { await editor.setPhpText('$test = 1;'); @@ -144,6 +165,21 @@ describe("Tree-sitter PHP grammar", () => { expect(editor).toHaveScopes([1, 0], 'final', ["keyword.control.final.php"]); expect(editor).toHaveScopes([1, 6], 'class', ["storage.type.class.php"]); }); + + describe('properties', () => { + it('scopes readonly properties', async () => { + await editor.setPhpText(` + class Test { + public readonly int $a; + }`) + + expect(editor).toHaveScopes([1, 0], 'class', ['storage.type.TYPE.php']) + expect(editor).toHaveScopes([2, 2], 'public', ['storage.modifier.public.php']) + expect(editor).toHaveScopes([2, 9], 'readonly', ['storage.modifier.readonly.php']) + expect(editor).toHaveScopes([2, 18], 'int', ['storage.type.builtin.php']) + }) + + }) }); describe("phpdoc", () => {