Add basic tests for PHP WASM grammar

packages/language-php/spec/tree-sitter-helpers.js

@@ -0,0 +1,171 @@
+const dedent = require("dedent");
+
+module.exports = {
+  // Taken from Atom source at
+  // https://github.com/atom/atom/blob/b3d3a52d9e4eb41f33df7b91ad1f8a2657a04487/spec/tree-sitter-language-mode-spec.js#L47-L55
+  // Not used in tests, but included for reference. I recall that it works by
+  // tokenizing lines and then lising the scopes for each token. This allows
+  // specs like:
+  //
+  // editor.setPhpText(`
+  // $foo + 1;
+  // $bar->baz;
+  // `)
+  // expectTokensToEqual(
+  //   editor,
+  //   [
+  //     [
+  //       {text: '$foo', scopes: [...]}
+  //       {text: '+', scopes: [...]}
+  //       {text: '1', scopes: [...]}
+  //     ],
+  //     [
+  //       {text: '$bar', scopes: [...]}
+  //       {text: '->',   scopes: [...]}
+  //       {text: 'baz', scopes: [...]}
+  //     ]
+  //   ]
+  // )
+  expectTokensToEqual(editor, expectedTokenLines, startingRow = 1) {
+    const lastRow = editor.getLastScreenRow();
+
+    for (let row = startingRow; row <= lastRow - startingRow; row++) {
+      const tokenLine = editor
+        .tokensForScreenRow(row)
+        .map(({ text, scopes }) => ({
+          text,
+          scopes: scopes.map((scope) =>
+            scope
+              .split(" ")
+              .map((className) => className.replace("syntax--", ""))
+              .join(".")
+          ),
+        }));
+
+      const expectedTokenLine = expectedTokenLines[row - startingRow];
+
+      expect(tokenLine.length).toEqual(expectedTokenLine.length);
+      for (let i = 0; i < tokenLine.length; i++) {
+        expect(tokenLine[i].text).toEqual(
+          expectedTokenLine[i].text,
+          `Token ${i}, row: ${row}`
+        );
+        expect(tokenLine[i].scopes).toEqual(
+          expectedTokenLine[i].scopes,
+          `Token ${i}, row: ${row}, token: '${tokenLine[i].text}'`
+        );
+      }
+    }
+  },
+
+  /**
+   * A matcher to compare scopes applied by a tree-sitter grammar on a character
+   * by character basis.
+   *
+   * @param  {array}        posn     Buffer position to be examined. A Point in the form [row, col]. Both are 0 based.
+   * @param  {string|array} token    The token to be matched at the given position. Mostly just to make the tests easier to read.
+   * @param  {?array}       expected The scopes that should be present.
+   * @param  {Object}       options  Options to change what is asserted.
+   */
+  toHaveScopes(posn, token, expected, options = {}) {
+    if (token === undefined) {
+        throw new Error(
+            'toHaveScopes must be called with at least 2 parameters'
+        );
+    }
+    if (expected === undefined) {
+      expected = token;
+      token = '';
+    }
+
+    // remove base scopes by default
+    const removeBaseScopes = options.removeBaseScopes ?? true;
+    const filterBaseScopes = (scope) =>
+      (
+        removeBaseScopes &&
+        scope !== "text.html.php" &&
+        scope !== "source.php"
+      );
+
+    // this.actual is a Pulsar TextEditor
+    const line = this.actual.getBuffer().lineForRow(posn[0]);
+    const caret = " ".repeat(posn[1]) + "^";
+
+    const actualToken = this.actual
+        .getTextInBufferRange([posn, [posn[0], posn[1] + token.length]]);
+
+    if (actualToken !== token) {
+        this.message = () => `
+  Failure: Tokens did not match at position [${posn.join(", ")}]:
+${line}
+${caret}
+  Expected token: ${token}
+`
+        return false;
+    }
+
+    const actualScopes = this.actual
+      .scopeDescriptorForBufferPosition(posn)
+      .scopes
+      .filter(filterBaseScopes);
+
+    const notExpected = actualScopes.filter((scope) => !expected.includes(scope));
+    const notReceived = expected.filter((scope) => !actualScopes.includes(scope));
+
+    const pass = notExpected.length === 0 && notReceived.length === 0;
+
+    if (pass) {
+      this.message = () => "Scopes matched";
+      return true;
+    }
+
+    this.message = () =>
+      `
+  Failure: Scopes did not match at position [${posn.join(", ")}]:
+${line}
+${caret}
+  These scopes were expected but not received:
+      ${notReceived.join(", ")}
+  These scopes were received but not expected:
+      ${notExpected.join(", ")}
+      ` +
+      (
+        (options.showAllScopes ?? false)
+        ? `
+  These were all scopes recieved:
+      ${actualScopes.join(", ")}
+  These were all scopes expected:
+      ${expected.join(", ")}
+        `
+        : ''
+     );
+
+    return false;
+  },
+
+  /**
+   * Wrap a code snippet in PHP tags, insert it into an editor, and wait for the
+   * language mode to be ready.
+   * @param  {string}  content a PHP code snippet
+   * @return {Promise}         resolves when the editor language mode is ready
+   */
+  async setPhpText(content) {
+    this.setText(`<?php
+${dedent(content)}
+`);
+    await this.languageMode.ready
+  },
+
+  // currently unused; may only be needed for legacy tree-sitter grammars?
+  nextHighlightingUpdate(editor) {
+    return new Promise((resolve) => {
+      const subscription = editor
+        .getBuffer()
+        .getLanguageMode()
+        .onDidChangeHighlighting(() => {
+          subscription.dispose();
+          resolve();
+        });
+    });
+  },
+};

packages/language-php/spec/wasm-tree-sitter-spec.js

@@ -0,0 +1,198 @@
+const {toHaveScopes, setPhpText} = require('./tree-sitter-helpers')
+
+describe("Tree-sitter PHP grammar", () => {
+  var editor;
+
+  beforeEach(async () => {
+    atom.config.set("core.useTreeSitterParsers", true);
+    atom.config.set("core.useExperimentalModernTreeSitter", true);
+    await atom.packages.activatePackage("language-php");
+    editor = await atom.workspace.open("foo.php");
+    editor.setPhpText = setPhpText;
+  });
+
+  beforeEach(function () {
+    this.addMatchers({toHaveScopes});
+  });
+
+  describe("loading the grammars", () => {
+    it('loads the wrapper "HTML" grammar', () => {
+      embeddingGrammar = atom.grammars.grammarForScopeName("text.html.php");
+      expect(embeddingGrammar).toBeTruthy();
+      expect(embeddingGrammar.scopeName).toBe("text.html.php");
+      expect(embeddingGrammar.constructor.name).toBe("WASMTreeSitterGrammar");
+
+      // injections
+      expect(embeddingGrammar.injectionPointsByType.program).toBeTruthy();
+      expect(embeddingGrammar.injectionPointsByType.comment).toBeTruthy();
+    })
+  });
+
+  describe('php tags', () => {
+    it('scopes regular opening tags', async () => {
+        await editor.setPhpText('');
+
+        expect(editor).toHaveScopes([0, 0], '<?php', ['punctuation.section.embedded.begin.php']);
+    })
+
+    it('scopes short opening tags', async () => {
+        editor.setText('<?= ?>');
+        await editor.languageMode.ready;
+
+        expect(editor).toHaveScopes([0, 0], '<?=', ['punctuation.section.embedded.begin.php']);
+    })
+
+    it('scopes closing tags', async () => {
+        await editor.setPhpText('?>');
+
+        expect(editor).toHaveScopes([1, 0], '?>', ['punctuation.section.embedded.end.php']);
+    })
+  });
+
+  describe("operators", () => {
+    it("scopes =", async () => {
+      await editor.setPhpText('$test = 1;');
+
+      expect(editor).toHaveScopes([1, 0], '$', ["variable.other.php", "punctuation.definition.variable.php"]);
+      expect(editor).toHaveScopes([1, 5], ' ', []);
+      expect(editor).toHaveScopes([1, 6], '=', ["keyword.operator.assignment.php"]);
+      expect(editor).toHaveScopes([1, 8], '1', ["constant.numeric.decimal.integer.php"]);
+    });
+
+    it("scopes +", async () => {
+      await editor.setPhpText('1 + 2;');
+
+      expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.integer.php"]);
+      expect(editor).toHaveScopes([1, 2], '+', ["keyword.operator.arithmetic.php"]);
+      expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.integer.php"]);
+    });
+
+    it("scopes %", async () => {
+      await editor.setPhpText('1 % 2;');
+
+      expect(editor).toHaveScopes([1, 0], '1', ["constant.numeric.decimal.integer.php"]);
+      expect(editor).toHaveScopes([1, 2], '%', ["keyword.operator.arithmetic.php"]);
+      expect(editor).toHaveScopes([1, 4], '2', ["constant.numeric.decimal.integer.php"]);
+    });
+
+    it("scopes instanceof", async () => {
+      await editor.setPhpText('$x instanceof Foo;');
+
+      expect(editor).toHaveScopes([1, 0],  '$',          ["variable.other.php", "punctuation.definition.variable.php"]);
+      expect(editor).toHaveScopes([1, 1],  'x',          ["variable.other.php"]);
+      expect(editor).toHaveScopes([1, 3],  'instanceof', ["keyword.operator.type.php"]);
+      expect(editor).toHaveScopes([1, 14], 'Foo',        ["support.other.function.constructor.php"]);
+    });
+
+    describe("combined operators", () => {
+      it("scopes ===", async () => {
+        await editor.setPhpText('$test === 2;');
+
+        expect(editor).toHaveScopes([1, 6], ["keyword.operator.comparison.php"]);
+      });
+
+      it("scopes +=", async () => {
+        await editor.setPhpText('$test += 2;');
+
+        expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.compound.php"]);
+      });
+
+      it("scopes ??=", async () => {
+        await editor.setPhpText('$test ??= true;');
+
+        expect(editor).toHaveScopes([1, 6], ["keyword.operator.assignment.compound.php"]);
+      });
+    });
+  });
+
+  it("scopes $this", async () => {
+    await editor.setPhpText("$this;");
+
+    expect(editor).toHaveScopes([1, 0], '$',    ["variable.language.builtin.this.php", "punctuation.definition.variable.php"]);
+    expect(editor).toHaveScopes([1, 1], 'this', ["variable.language.builtin.this.php"]);
+
+    await editor.setPhpText("$thistles;");
+
+    expect(editor).toHaveScopes([1, 0], '$',        ["variable.other.php", "punctuation.definition.variable.php"]);
+    expect(editor).toHaveScopes([1, 1], 'thistles', ["variable.other.php"]);
+  });
+
+  describe("use declarations", () => {
+    it("scopes basic use statements", async () => {
+      await editor.setPhpText("use Foo;");
+
+      expect(editor).toHaveScopes([1, 0], 'use', ["keyword.control.use.php"]);
+      expect(editor).toHaveScopes([1, 4], 'Foo', ["entity.name.type.namespace.php"]);
+
+      await editor.setPhpText("use My\\Full\\NSname;");
+
+      expect(editor).toHaveScopes([1, 0], 'use', ["keyword.control.use.php"]);
+      expect(editor).toHaveScopes([1, 4], 'My', ["entity.name.type.namespace.php"]);
+      expect(editor).toHaveScopes([1, 6], '\\', ["punctuation.operator.namespace.php"]);
+      expect(editor).toHaveScopes([1, 7], 'Full', ["entity.name.type.namespace.php"]);
+      expect(editor).toHaveScopes([1, 11], '\\', ["punctuation.operator.namespace.php"]);
+      expect(editor).toHaveScopes([1, 12], 'NSname', ["entity.name.type.namespace.php"]);
+    });
+  });
+
+  describe("classes", () => {
+    it("scopes class declarations", async () => {
+      await editor.setPhpText("class Test {}");
+
+      expect(editor).toHaveScopes([1,  0], 'class', ["storage.type.class.php"]);
+      expect(editor).toHaveScopes([1,  6], 'Test',  ["entity.name.type.class.php"]);
+      expect(editor).toHaveScopes([1, 11], '{',     ['punctuation.definition.block.begin.bracket.curly.php']);
+      expect(editor).toHaveScopes([1, 12], '}',     ['punctuation.definition.block.end.bracket.curly.php']);
+    });
+
+    it("scopes class instantiation", async () => {
+      await editor.setPhpText("$a = new ClassName();");
+
+      expect(editor).toHaveScopes([1, 5], 'new',       ["keyword.control.new.php"]);
+      expect(editor).toHaveScopes([1, 9], 'ClassName', ["support.other.function.constructor.php"]);
+      expect(editor).toHaveScopes([1, 18], '(',        ["punctuation.definition.begin.bracket.round.php"]);
+      expect(editor).toHaveScopes([1, 19], ')',        ["punctuation.definition.end.bracket.round.php"]);
+    });
+
+    it("scopes class modifiers", async () => {
+      await editor.setPhpText("abstract class Test {}");
+
+      expect(editor).toHaveScopes([1, 0], 'abstract', ["keyword.control.abstract.php"]);
+      expect(editor).toHaveScopes([1, 9], 'class', ["storage.type.class.php"]);
+
+      await editor.setPhpText("final class Test {}");
+
+      expect(editor).toHaveScopes([1, 0], 'final', ["keyword.control.final.php"]);
+      expect(editor).toHaveScopes([1, 6], 'class', ["storage.type.class.php"]);
+    });
+
+    describe('properties', () => {
+      it('scopes readonly properties', async () => {
+        await editor.setPhpText(`
+          class Test {
+            public readonly int $a;
+          }`)
+
+          expect(editor).toHaveScopes([1, 0], 'class',    ['storage.type.TYPE.php'])
+          expect(editor).toHaveScopes([2, 2], 'public',   ['storage.modifier.public.php'])
+          expect(editor).toHaveScopes([2, 9], 'readonly', ['storage.modifier.readonly.php'])
+          expect(editor).toHaveScopes([2, 18], 'int',     ['storage.type.builtin.php'])
+        })
+
+    })
+  });
+
+  describe("phpdoc", () => {
+    it("scopes @return tags", async () => {
+      await editor.setPhpText("/** @return Foo<Bar> */");
+
+      expect(editor).toHaveScopes([1,  0], '/**',     ['comment.block.documentation.phpdoc.php', 'punctuation.definition.begin.comment.phpdoc.php']);
+      expect(editor).toHaveScopes([1,  4], '@return', ['comment.block.documentation.phpdoc.php', 'entity.name.tag.phpdoc.php']);
+      expect(editor).toHaveScopes([1, 12], 'Foo',     ['comment.block.documentation.phpdoc.php', 'storage.type.instance.phpdoc.php']);
+      expect(editor).toHaveScopes([1, 15], '<',       ['comment.block.documentation.phpdoc.php']);
+      expect(editor).toHaveScopes([1, 16], 'Bar',     ['comment.block.documentation.phpdoc.php', 'storage.type.instance.phpdoc.php']);
+      expect(editor).toHaveScopes([1, 19], '>',       ['comment.block.documentation.phpdoc.php']);
+      expect(editor).toHaveScopes([1, 21], '*/',      ['comment.block.documentation.phpdoc.php', 'punctuation.definition.end.comment.phpdoc.php']);
+    });
+  });
+});