diff --git a/.eslintrc.js b/.eslintrc.js index 08b4fff876..41df8c5812 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -19,6 +19,14 @@ module.exports = { asyncArrow: "always", named: "never" }], + "no-constant-condition": "off", + "no-unused-vars": [ + "warn", + { + varsIgnorePattern: '^_', + argsIgnorePattern: '^_' + } + ], "node/no-missing-require": [ "error", { diff --git a/packages/language-c/settings/language-c.cson b/packages/language-c/settings/language-c.cson index 17d33e4173..355b5c88a4 100644 --- a/packages/language-c/settings/language-c.cson +++ b/packages/language-c/settings/language-c.cson @@ -1,6 +1,11 @@ '.source.c, .source.cpp, .source.objc, .source.objcpp': 'editor': 'commentStart': '// ' + # Technically, line comments aren't universally valid in C, but all modern + # C compilers support them. + 'commentDelimiters': + 'block': ['/*', '*/'] + 'line': '//' 'increaseIndentPattern': '(?x) ^ .* \\{ [^}"\']* $ |^ .* \\( [^\\)"\']* $ diff --git a/packages/language-clojure/settings/language-clojure.cson b/packages/language-clojure/settings/language-clojure.cson index d0dd7180e8..5df9983685 100644 --- a/packages/language-clojure/settings/language-clojure.cson +++ b/packages/language-clojure/settings/language-clojure.cson @@ -1,5 +1,7 @@ '.source.clojure': 'editor': 'commentStart': '; ' + 'commentDelimiters': + 'line': ';' 'autocomplete': 'extraWordCharacters': '-' diff --git a/packages/language-coffee-script/settings/language-coffee-script.cson b/packages/language-coffee-script/settings/language-coffee-script.cson index 9aeb77a27c..ec18059ae9 100644 --- a/packages/language-coffee-script/settings/language-coffee-script.cson +++ b/packages/language-coffee-script/settings/language-coffee-script.cson @@ -1,6 +1,8 @@ '.source.coffee, .source.litcoffee, .source.coffee.md': 'editor': 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' '.source.coffee': 'editor': 'autoIndentOnPaste': false diff --git a/packages/language-csharp/settings/language-csharp.cson b/packages/language-csharp/settings/language-csharp.cson index 970f58082d..75fa6ae4ba 100644 --- a/packages/language-csharp/settings/language-csharp.cson +++ b/packages/language-csharp/settings/language-csharp.cson @@ -1,5 +1,8 @@ '.source.cs': 'editor': 'commentStart': '// ' + 'commentDelimiters': + 'line': '//' + 'block': ['/*', '*/'] 'increaseIndentPattern': '(?x)\n\t\t^ .* \\{ [^}"\']* $\n\t| ^ \\s* \\{ \\} $\n\t' 'decreaseIndentPattern': '(?x)\n\t\t^ (.*\\*/)? \\s* \\} ( [^}{"\']* \\{ | \\s* while \\s* \\( .* )? [;\\s]* (//.*|/\\*.*\\*/\\s*)? $\n\t' diff --git a/packages/language-css/settings/language-css.cson b/packages/language-css/settings/language-css.cson index bb57b398cc..6484815245 100644 --- a/packages/language-css/settings/language-css.cson +++ b/packages/language-css/settings/language-css.cson @@ -4,6 +4,8 @@ 'editor': 'commentStart': '/*' 'commentEnd': '*/' + 'commentDelimiters': + 'block': ['/*', '*/'] 'foldEndPattern': '(?' + 'commentDelimiters': + 'block': [''] diff --git a/packages/language-go/settings/language-go.cson b/packages/language-go/settings/language-go.cson index f6ebb10ca1..bb330b5b7c 100644 --- a/packages/language-go/settings/language-go.cson +++ b/packages/language-go/settings/language-go.cson @@ -1,6 +1,9 @@ '.source.go': 'editor': 'commentStart': '// ' + 'commentDelimiters': + 'block': ['/*', '*/'] + 'line': '//' 'increaseIndentPattern': '^.*(\\bcase\\b.*:|\\bdefault\\b:|(\\b(func|if|else|switch|select|for|struct)\\b.*)?{[^}]*|\\([^)]*)$' 'decreaseIndentPattern': '^\\s*(\\bcase\\b.*:|\\bdefault\\b:|}[),]?|\\)[,]?)$' 'decreaseNextIndentPattern': '^\\s*[^\\s()}]+(?[^()]*\\((?:\\g[^()]*|[^()]*)\\))*[^()]*\\)[,]?$' diff --git a/packages/language-html/grammars/modern-tree-sitter-html.cson b/packages/language-html/grammars/modern-tree-sitter-html.cson index fa5fcc41cb..55b856d1c7 100644 --- a/packages/language-html/grammars/modern-tree-sitter-html.cson +++ b/packages/language-html/grammars/modern-tree-sitter-html.cson @@ -21,3 +21,4 @@ fileTypes: [ comments: start: '' + block: [''] diff --git a/packages/language-html/settings/language-html.cson b/packages/language-html/settings/language-html.cson index 1b46997d57..0f326cfdf6 100644 --- a/packages/language-html/settings/language-html.cson +++ b/packages/language-html/settings/language-html.cson @@ -4,6 +4,8 @@ 'editor': 'commentStart': '' + 'commentDelimiters': + 'block': [''] 'foldEndPattern': '(?x)\n\t\t(\n\t\t|^(?!.*?$\n\t\t|<\\?(?:php)?.*\\bend(if|for(each)?|while)\\b\n\t\t|\\{\\{?/(if|foreach|capture|literal|foreach|php|section|strip)\n\t\t|^[^{]*\\}\n\t\t|^\\s*\\)[,;]\n\t\t)' 'increaseIndentPattern': '''(?x) <(?!\\?|(?:area|base|br|col|frame|hr|html|img|input|link|meta|param)\\b|[^>]*/>) diff --git a/packages/language-java/settings/language-java.cson b/packages/language-java/settings/language-java.cson index 49bda36bd1..ae503b962d 100644 --- a/packages/language-java/settings/language-java.cson +++ b/packages/language-java/settings/language-java.cson @@ -1,6 +1,9 @@ '.source.java': 'editor': 'commentStart': '// ' + 'commentDelimiters': + 'block': ['/*', '*/'] + 'line': '//' 'foldEndPattern': '^\\s*(\\}|// \\}\\}\\}$)' 'increaseIndentPattern': '^.*\\{(\\}|[^}"\']*)$|^\\s*(public|private|protected):\\s*$' 'decreaseIndentPattern': '^(.*\\*/)?\\s*\\}|^\\s*(public|private|protected):\\s*$' @@ -9,6 +12,8 @@ 'foldEndPattern': '\\*\\*/|^\\s*\\}' 'commentStart': '<%-- ' 'commentEnd': ' --%>' + 'commentDelimiters': + 'block': ['<%--', '-->'] 'increaseIndentPattern': '^\\s*<(([^!/?]|%)(?!.+?([/%]>|))|[%!]--\\s*$)' 'decreaseIndentPattern': '^\\s*(]+>|-->|--%>)' '.text.junit-test-report': @@ -17,6 +22,8 @@ '.source.java-properties': 'editor': 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' '.keyword.other.documentation.javadoc.java': 'editor': 'completions': [ diff --git a/packages/language-javascript/grammars/modern-tree-sitter-javascript.cson b/packages/language-javascript/grammars/modern-tree-sitter-javascript.cson index b2fc9fc6ab..084577fd01 100644 --- a/packages/language-javascript/grammars/modern-tree-sitter-javascript.cson +++ b/packages/language-javascript/grammars/modern-tree-sitter-javascript.cson @@ -31,3 +31,5 @@ fileTypes: [ comments: start: '// ' + block: ['/*', '*/'] + line: '//' diff --git a/packages/language-javascript/settings/language-javascript.cson b/packages/language-javascript/settings/language-javascript.cson index 48b832fb6e..5684fdeaad 100644 --- a/packages/language-javascript/settings/language-javascript.cson +++ b/packages/language-javascript/settings/language-javascript.cson @@ -2,6 +2,9 @@ 'editor': 'nonWordCharacters': '/\\()"\':,.;<>~!#@%^&*|+=[]{}`?-…' 'commentStart': '// ' + 'commentDelimiters': + 'block': ['/*', '*/'] + 'line': '//' 'foldEndPattern': '^\\s*\\}|^\\s*\\]|^\\s*\\)' 'increaseIndentPattern': '(?x) \\{ [^}"\']*(//.*)? $ @@ -16,8 +19,12 @@ 'editor': 'commentStart': '{/*' 'commentEnd': '*/}' + 'commentDelimiters': + 'block': ['{/*', '*/}'] '.source.js .meta.jsx.inside-tag.js': 'editor': 'commentStart': '{/*' 'commentEnd': '*/}' + 'commentDelimiters': + 'block': ['{/*', '*/}'] diff --git a/packages/language-less/settings/language-less.cson b/packages/language-less/settings/language-less.cson index 83f17a2dd4..286d48bc98 100644 --- a/packages/language-less/settings/language-less.cson +++ b/packages/language-less/settings/language-less.cson @@ -1,6 +1,9 @@ ".source.css.less": editor: commentStart: "// " + commentDelimiters: + line: '//' + block: ['/*', '*/'] autocomplete: extraWordCharacters: '-' symbols: diff --git a/packages/language-make/settings/language-make.cson b/packages/language-make/settings/language-make.cson index 3a31f1b430..43dd209359 100644 --- a/packages/language-make/settings/language-make.cson +++ b/packages/language-make/settings/language-make.cson @@ -2,4 +2,6 @@ 'editor': 'increaseIndentPattern': '^[^\\t ]+:' 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' 'tabType': 'hard' diff --git a/packages/language-objective-c/settings/language-objective-c.cson b/packages/language-objective-c/settings/language-objective-c.cson index da582be8c7..88e34f0a72 100644 --- a/packages/language-objective-c/settings/language-objective-c.cson +++ b/packages/language-objective-c/settings/language-objective-c.cson @@ -32,6 +32,7 @@ 'NSMutableString' 'NSString' ] + # Comment settings are specified in the `language-c` package. '.source.objcpp, .source.objc': 'editor': 'foldEndPattern': '(?~!#^&*|+=[]{}`?-' '.source.perl6': 'editor': @@ -11,4 +13,6 @@ 'increaseIndentPattern': '^.*\\{\\}?\\s*$' 'decreaseIndentPattern': '^\\s*\\}' 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' 'nonWordCharacters': '/\\()"\':,.;<>~!#^&*|+=[]{}`?-' diff --git a/packages/language-php/settings/language-php.cson b/packages/language-php/settings/language-php.cson index 15e9ac9aad..652bbf17da 100644 --- a/packages/language-php/settings/language-php.cson +++ b/packages/language-php/settings/language-php.cson @@ -1,6 +1,9 @@ '.source.php': 'editor': 'commentStart': '// ' + 'commentDelimiters': + 'line': '//' + 'block': ['/*', '*/'] 'completions': [ 'APCIterator' 'AppendIterator' diff --git a/packages/language-property-list/settings/language-property-list.cson b/packages/language-property-list/settings/language-property-list.cson index d2db74d401..29389fd4e3 100644 --- a/packages/language-property-list/settings/language-property-list.cson +++ b/packages/language-property-list/settings/language-property-list.cson @@ -1,6 +1,8 @@ '.source.plist': 'editor': 'commentStart': '// ' + 'commentDelimiters': + 'line': '//' 'foldEndPattern': '^\\s*(\\}|\\))' 'increaseIndentPattern': '^\\s*(([a-zA-Z_-]+|"[^"]+"|\'[^\']+\')\\s+=\\s+)?[{(](?!.*[)}][;,]?\\s*$)' 'decreaseIndentPattern': '^\\s*(\\}|\\))' diff --git a/packages/language-python/settings/language-python.cson b/packages/language-python/settings/language-python.cson index 8538f101a4..0eb175b20a 100644 --- a/packages/language-python/settings/language-python.cson +++ b/packages/language-python/settings/language-python.cson @@ -4,6 +4,8 @@ 'softTabs': true 'tabLength': 4 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' 'foldEndPattern': '^\\s*[}\\])]' 'increaseIndentPattern': '^\\s*(class|def|elif|else|except|finally|for|if|try|with|while|async\\s+(def|for|with))\\b.*:\\s*$' 'decreaseIndentPattern': '^\\s*(elif|else|except|finally)\\b.*:\\s*$' diff --git a/packages/language-ruby/settings/language-ruby.cson b/packages/language-ruby/settings/language-ruby.cson index 1e72d9cd57..e24e6511e0 100644 --- a/packages/language-ruby/settings/language-ruby.cson +++ b/packages/language-ruby/settings/language-ruby.cson @@ -1,6 +1,8 @@ '.source.ruby': 'editor': 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' 'increaseIndentPattern': '(?x)^\n (\\s*\n (module|class|(private\\s+)?def\n |unless|if|else|elsif\n |case|when\n |begin|rescue|ensure\n |for|while|until\n |(?= .*? \\b(do|begin|case|if|unless)\\b )\n # the look-ahead above is to quickly discard non-candidates\n ( "(\\\\.|[^\\\\"])*+" # eat a double quoted string\n | \'(\\\\.|[^\\\\\'])*+\' # eat a single quoted string\n | [^#"\'] # eat all but comments and strings\n )*\n ( \\s (do|begin|case)\n | [-+=&|*/~%^<>~](?~!#%^&*|+=[]{}`?…' 'commentStart': '// ' + 'commentDelimiters': + 'line': '//' + 'block': ['/*', '*/'] 'autocomplete': 'symbols': 'mixin': diff --git a/packages/language-shellscript/settings/language-shellscript.cson b/packages/language-shellscript/settings/language-shellscript.cson index 0e39701320..4b4a9c8387 100644 --- a/packages/language-shellscript/settings/language-shellscript.cson +++ b/packages/language-shellscript/settings/language-shellscript.cson @@ -1,6 +1,8 @@ '.source.shell': 'editor': 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' 'foldEndPattern': '^\\s*(\\}|(done|fi|esac)\\b)' 'increaseIndentPattern': '^\\s*(else|case)\\b|^.*(\\{|\\b(then|do)\\b)$' 'decreaseIndentPattern': '^\\s*(\\}|(elif|else|fi|esac|done)\\b)' diff --git a/packages/language-sql/settings/language-sql.cson b/packages/language-sql/settings/language-sql.cson index d464309043..0ab0b9700a 100644 --- a/packages/language-sql/settings/language-sql.cson +++ b/packages/language-sql/settings/language-sql.cson @@ -1,6 +1,8 @@ '.source.sql': 'editor': 'commentStart': '-- ' + 'commentDelimiters': + 'line': '--' 'foldEndPattern': '^\\s*\\)' 'increaseIndentPattern': '^\\s*(create|grant|insert|delete|update)\\b|\\((?!.*\\))' 'decreaseIndentPattern': '^\\s*\\)(?!=.*\\()' diff --git a/packages/language-toml/settings/language-toml.cson b/packages/language-toml/settings/language-toml.cson index a748d5f929..3ae60a7964 100644 --- a/packages/language-toml/settings/language-toml.cson +++ b/packages/language-toml/settings/language-toml.cson @@ -1,3 +1,5 @@ '.source.toml': 'editor': 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' diff --git a/packages/language-typescript/settings/TypeScript.cson b/packages/language-typescript/settings/TypeScript.cson index 76a0e24a41..e7a21bfbf8 100644 --- a/packages/language-typescript/settings/TypeScript.cson +++ b/packages/language-typescript/settings/TypeScript.cson @@ -1,6 +1,9 @@ '.source.ts': 'editor': 'commentStart': '// ' + 'commentDelimiters': + 'line': '//' + 'block': ['/*', '*/'] 'foldEndPattern': '^\\s*\\}|^\\s*\\]|^\\s*\\)' 'increaseIndentPattern': '(?x) \\{ [^}"\']* $ diff --git a/packages/language-typescript/settings/TypeScriptReact.cson b/packages/language-typescript/settings/TypeScriptReact.cson index b99255b1f4..18a7638674 100644 --- a/packages/language-typescript/settings/TypeScriptReact.cson +++ b/packages/language-typescript/settings/TypeScriptReact.cson @@ -1,6 +1,9 @@ '.source.tsx': 'editor': 'commentStart': '// ' + 'commentDelimiters': + 'line': '//' + 'block': ['/*', '*/'] 'foldEndPattern': '^\\s*\\}|^\\s*\\]|^\\s*\\)' 'increaseIndentPattern': '(?x) \\{ [^}"\']* $ @@ -15,6 +18,8 @@ 'editor': 'commentStart': '{/* ', 'commentEnd': ' */}', + 'commentDelimiters': + 'block': ['{/*', '*/}'] 'increaseIndentPattern': "{[^}\"']*$|\\[[^\\]\"']*$|\\([^)\"']*$|<[a-zA-Z][^/]*$|^\\s*>$", 'decreaseIndentPattern': "^\\s*(\\s*/[*].*[*]/\\s*)*[}\\])]|^\\s*()" diff --git a/packages/language-xml/settings/language-xml.cson b/packages/language-xml/settings/language-xml.cson index 37403fa554..cdeabc868d 100644 --- a/packages/language-xml/settings/language-xml.cson +++ b/packages/language-xml/settings/language-xml.cson @@ -2,6 +2,8 @@ 'editor': 'commentStart': '' + 'commentDelimiters': + 'block': [''] 'foldEndPattern': '^\\s*(]+>|[/%]>|-->)\\s*$' 'increaseIndentPattern': '^\\s*<(([^!/?]|%)(?!.+?([/%]>|))|[%!]--\\s*$)' 'decreaseIndentPattern': '^\\s*(]+>|-->|--%>)' diff --git a/packages/language-yaml/settings/language-yaml.cson b/packages/language-yaml/settings/language-yaml.cson index 2fd8af7a2b..82ad135f91 100644 --- a/packages/language-yaml/settings/language-yaml.cson +++ b/packages/language-yaml/settings/language-yaml.cson @@ -2,6 +2,8 @@ 'editor': 'autoIndentOnPaste': false 'commentStart': '# ' + 'commentDelimiters': + 'line': '#' 'foldEndPattern': '^\\s*$|^\\s*\\}|^\\s*\\]|^\\s*\\)' 'increaseIndentPattern': '^\\s*.*(:|-) ?(&\\w+)?(\\{[^}"\']*|\\([^)"\']*)?$' 'decreaseIndentPattern': '^\\s+\\}$' diff --git a/spec/text-editor-spec.js b/spec/text-editor-spec.js index 2aaf4da24f..832c482a53 100644 --- a/spec/text-editor-spec.js +++ b/spec/text-editor-spec.js @@ -9,7 +9,7 @@ const TextBuffer = require('text-buffer'); const TextMateLanguageMode = require('../src/text-mate-language-mode'); const WASMTreeSitterLanguageMode = require('../src/wasm-tree-sitter-language-mode'); -async function languageModeReady (editor) { +async function languageModeReady(editor) { let languageMode = editor.getBuffer().getLanguageMode(); if (languageMode.ready) { await languageMode.ready; @@ -1586,7 +1586,7 @@ describe('TextEditor', () => { it("doesn't get stuck in a infinite loop when called from ::onDidAddCursor after the last selection has been destroyed (regression)", () => { let callCount = 0; editor.getLastSelection().destroy(); - editor.onDidAddCursor(function(cursor) { + editor.onDidAddCursor(function (cursor) { callCount++; editor.getLastSelection(); }); @@ -2250,7 +2250,7 @@ describe('TextEditor', () => { spyOn( editor.getBuffer().getLanguageMode(), 'getNonWordCharacters' - ).andCallFake(function(position) { + ).andCallFake(function (position) { const result = '/()"\':,.;<>~!@#$%^&*|+=[]{}`?'; const scopes = this.scopeDescriptorForPosition( position @@ -5313,7 +5313,7 @@ describe('TextEditor', () => { it('notifies ::onWillInsertText observers', () => { const insertedStrings = []; - editor.onWillInsertText(function({ text, cancel }) { + editor.onWillInsertText(function ({ text, cancel }) { insertedStrings.push(text); cancel(); }); @@ -8211,6 +8211,64 @@ describe('TextEditor', () => { }); }); + describe('.getCommentDelimitersForBufferPosition', () => { + it('returns comment delimiters on a TextMate grammar', async () => { + atom.config.set('core.useTreeSitterParsers', false); + + editor = await atom.workspace.open('sample.js', { autoIndent: false }); + await atom.packages.activatePackage('language-javascript'); + + let buffer = editor.getBuffer(); + + let languageMode = new TextMateLanguageMode({ + buffer, + grammar: atom.grammars.grammarForScopeName('source.js') + }); + + buffer.setLanguageMode(languageMode); + + languageMode.startTokenizing(); + while (languageMode.firstInvalidRow() != null) { + advanceClock(); + } + + let delimiters = editor.getCommentDelimitersForBufferPosition([8, 0]); + expect(delimiters).toEqual({ + line: '//', + block: ['/*', '*/'] + }) + }) + + it('returns comment delimiters on a modern Tree-sitter grammar', async () => { + jasmine.useRealClock(); + atom.config.set('core.useTreeSitterParsers', true); + + editor = await atom.workspace.open('sample.js', { autoIndent: false }); + await atom.packages.activatePackage('language-javascript'); + + let buffer = editor.getBuffer(); + + let languageMode = new WASMTreeSitterLanguageMode({ + buffer, + grammar: atom.grammars.grammarForScopeName('source.js'), + grammars: atom.grammars + }); + + languageMode.useAsyncParsing = false; + languageMode.useAsyncIndent = false; + + buffer.setLanguageMode(languageMode); + await languageMode.ready; + + + let delimiters = editor.getCommentDelimitersForBufferPosition([8, 0]); + expect(delimiters).toEqual({ + line: '//', + block: ['/*', '*/'] + }) + }) + }) + describe('.syntaxTreeScopeDescriptorForBufferPosition(position)', () => { it('returns the result of scopeDescriptorForBufferPosition() when textmate language mode is used', async () => { atom.config.set('core.useTreeSitterParsers', false); diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 889e7e30a7..3fc7451d78 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -87,7 +87,7 @@ describe('TreeSitterLanguageMode', () => { }); const original = grammar.idForScope.bind(grammar); let tokens = []; - grammar.idForScope = function(scope, text) { + grammar.idForScope = function (scope, text) { if (text && tokens[tokens.length - 1] !== text) { tokens.push(text); } @@ -2342,13 +2342,30 @@ describe('TreeSitterLanguageMode', () => { const htmlCommentStrings = { commentStartString: '' + commentEndString: '-->', + commentDelimiters: { + line: undefined, + block: [''] + } }; const jsCommentStrings = { commentStartString: '//', - commentEndString: undefined + commentEndString: undefined, + commentDelimiters: { + line: '//', + block: undefined + } }; + const hybridCommentStrings = { + commentStartString: '//', + commentEndString: undefined, + commentDelimiters: { + line: undefined, + block: [''] + } + } + expect(languageMode.commentStringsForPosition(new Point(0, 0))).toEqual( htmlCommentStrings ); @@ -2356,16 +2373,16 @@ describe('TreeSitterLanguageMode', () => { htmlCommentStrings ); expect(languageMode.commentStringsForPosition(new Point(2, 0))).toEqual( - jsCommentStrings + hybridCommentStrings ); expect(languageMode.commentStringsForPosition(new Point(3, 0))).toEqual( - jsCommentStrings + hybridCommentStrings ); expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual( htmlCommentStrings ); expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual( - jsCommentStrings + hybridCommentStrings ); expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual( htmlCommentStrings diff --git a/spec/wasm-tree-sitter-language-mode-spec.js b/spec/wasm-tree-sitter-language-mode-spec.js index 8b7daf33a4..0d4836f8b7 100644 --- a/spec/wasm-tree-sitter-language-mode-spec.js +++ b/spec/wasm-tree-sitter-language-mode-spec.js @@ -3238,6 +3238,15 @@ describe('WASMTreeSitterLanguageMode', () => { }); describe('.commentStringsForPosition(position)', () => { + beforeEach(() => { + atom.config.unset('editor.commentDelimiters', { scopeSelector: '.source.js' }); + atom.config.unset('editor.commentStart', { scopeSelector: '.source.js' }); + atom.config.unset('editor.commentEnd', { scopeSelector: '.source.js' }); + atom.config.unset('editor.commentDelimiters', { scopeSelector: '.text.html.basic' }); + atom.config.unset('editor.commentStart', { scopeSelector: '.text.html.basic' }); + atom.config.unset('editor.commentEnd', { scopeSelector: '.text.html.basic' }); + }); + it('returns the correct comment strings for nested languages', async () => { jasmine.useRealClock(); const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); @@ -3255,6 +3264,42 @@ describe('WASMTreeSitterLanguageMode', () => { atom.grammars.addGrammar(jsGrammar); atom.grammars.addGrammar(htmlGrammar); + atom.config.set( + 'editor.commentDelimiters', + { + line: '//', + block: ['/*', '*/'] + }, + { scopeSelector: '.source.js' } + ); + + atom.config.set( + 'editor.commentStart', + '//', + { scopeSelector: '.source.js' } + ); + + + atom.config.set( + 'editor.commentDelimiters', + { + block: [''] + }, + { scopeSelector: '.text.html.basic' } + ); + + atom.config.set( + 'editor.commentStart', + '', + { scopeSelector: '.text.html.basic' } + ); + const languageMode = new WASMTreeSitterLanguageMode({ grammar: htmlGrammar, buffer, @@ -3277,12 +3322,20 @@ describe('WASMTreeSitterLanguageMode', () => { ); const htmlCommentStrings = { - commentStartString: '' + commentStartString: '', + commentDelimiters: { + line: undefined, + block: [''] + } }; const jsCommentStrings = { - commentStartString: '// ', - commentEndString: undefined + commentStartString: '//', + commentEndString: undefined, + commentDelimiters: { + line: '//', + block: ['/*', '*/'] + } }; // Needs a short delay to allow injection grammars to be loaded. @@ -3303,13 +3356,223 @@ describe('WASMTreeSitterLanguageMode', () => { expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual( htmlCommentStrings ); - expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual( + expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual({ + // This is the curveball. Original position is inside the HTML, so + // that's what the delimiters will be. But `commentStartString` will be + // `// ` because it looks up the scope of the first non-whitespace + // content on the row. + commentStartString: '//', + commentEndString: undefined, + commentDelimiters: { + line: undefined, + block: [''] + } + }); + expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual( + htmlCommentStrings + ); + }); + + it('uses grammar comment settings when config data is missing', async () => { + jasmine.useRealClock(); + const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); + + jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT); + + const htmlGrammar = new WASMTreeSitterGrammar( + atom.grammars, + htmlGrammarPath, + htmlConfig + ); + + htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT); + + atom.grammars.addGrammar(jsGrammar); + atom.grammars.addGrammar(htmlGrammar); + + const languageMode = new WASMTreeSitterLanguageMode({ + grammar: htmlGrammar, + buffer, + config: atom.config, + grammars: atom.grammars + }); + buffer.setLanguageMode(languageMode); + await languageMode.ready; + + buffer.setText( + ` +
hi
+ + `.trim() + ); + + const htmlCommentStrings = { + commentStartString: '', + commentDelimiters: { + line: undefined, + block: [''] + } + }; + const jsCommentStrings = { + commentStartString: '//', + commentEndString: undefined, + commentDelimiters: { + line: '//', + block: ['/*', '*/'] + } + }; + + // Needs a short delay to allow injection grammars to be loaded. + await languageMode.nextTransaction; + + expect(languageMode.commentStringsForPosition(new Point(0, 0))).toEqual( + htmlCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(1, 0))).toEqual( + htmlCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(2, 0))).toEqual( jsCommentStrings ); + expect(languageMode.commentStringsForPosition(new Point(3, 0))).toEqual( + jsCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual( + htmlCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual({ + // This is the curveball. Original position is inside the HTML, so + // that's what the delimiters will be. But `commentStartString` will be + // `// ` because it looks up the scope of the first non-whitespace + // content on the row. + commentStartString: '//', + commentEndString: undefined, + commentDelimiters: { + line: undefined, + block: [''] + } + }); + expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual( + htmlCommentStrings + ); + }); + + it('constructs the right comment settings when grammar data is missing', async () => { + jasmine.useRealClock(); + const jsGrammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); + + jsGrammar.addInjectionPoint(HTML_TEMPLATE_LITERAL_INJECTION_POINT); + + const htmlGrammar = new WASMTreeSitterGrammar( + atom.grammars, + htmlGrammarPath, + htmlConfig + ); + + spyOn(jsGrammar, 'getCommentDelimiters').andReturn({ line: undefined, block: undefined }); + spyOn(htmlGrammar, 'getCommentDelimiters').andReturn({ line: undefined, block: undefined }); + + atom.config.set( + 'editor.commentDelimiters', + { + line: '//', + block: ['/*', '*/'] + }, + { scopeSelector: '.source.js' } + ); + + atom.config.set( + 'editor.commentDelimiters', + { + block: [''] + }, + { scopeSelector: '.text.html.basic' } + ); + + htmlGrammar.addInjectionPoint(SCRIPT_TAG_INJECTION_POINT); + + atom.grammars.addGrammar(jsGrammar); + atom.grammars.addGrammar(htmlGrammar); + + const languageMode = new WASMTreeSitterLanguageMode({ + grammar: htmlGrammar, + buffer, + config: atom.config, + grammars: atom.grammars + }); + buffer.setLanguageMode(languageMode); + await languageMode.ready; + + buffer.setText( + ` +
hi
+ + `.trim() + ); + + const htmlCommentStrings = { + commentStartString: '', + commentDelimiters: { + line: undefined, + block: [''] + } + }; + const jsCommentStrings = { + commentStartString: '//', + commentEndString: undefined, + commentDelimiters: { + line: '//', + block: ['/*', '*/'] + } + }; + + // Needs a short delay to allow injection grammars to be loaded. + await languageMode.nextTransaction; + + expect(languageMode.commentStringsForPosition(new Point(0, 0))).toEqual( + htmlCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(1, 0))).toEqual( + htmlCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(2, 0))).toEqual( + jsCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(3, 0))).toEqual( + jsCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(4, 0))).toEqual( + htmlCommentStrings + ); + expect(languageMode.commentStringsForPosition(new Point(5, 0))).toEqual({ + // This is the curveball. Original position is inside the HTML, so + // that's what the delimiters will be. But `commentStartString` will be + // `// ` because it looks up the scope of the first non-whitespace + // content on the row. + commentStartString: '//', + commentEndString: undefined, + commentDelimiters: { + line: undefined, + block: [''] + } + }); expect(languageMode.commentStringsForPosition(new Point(6, 0))).toEqual( htmlCommentStrings ); }); + }); describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => { @@ -3320,11 +3583,6 @@ describe('WASMTreeSitterLanguageMode', () => { (program) @source `); - // { - // parser: 'tree-sitter-javascript', - // scopes: { program: 'source' } - // }); - buffer.setText(dedent` function a (b, c, d) { eee.f() diff --git a/src/comment-utils.js b/src/comment-utils.js new file mode 100644 index 0000000000..bf6a016829 --- /dev/null +++ b/src/comment-utils.js @@ -0,0 +1,85 @@ + +// Accept a comment metadata block and make it more consistent and predictable. +// +// The `comments` key in a grammar definition file has historically contained +// some comment metadata under the `start` and `end` properties, but only as +// much as was needed for the “Toggle Line Comment” command. Newer convention +// is to define `line` (`string`) and `block` (`[string, string]`) properties +// to reflect all comment metadata for the language. +// +// Some grammars also specify block comments as `{ start: string, end: string +// }`, so we'll normalize that to the `[string, string]` variant. +function normalizeDelimiters(meta = {}) { + // Adapt the style used in `TreeSitterGrammar`. + if ( + ('commentStartString' in meta && 'commentEndString' in meta) && !('line' in meta || 'block' in meta) + ) { + let { commentStartString: start, commentEndString: end } = meta; + meta = { start, end }; + } + let { line, block } = meta; + // Normalize the `{ start: string, end: string }` version to `[string, + // string].` + if (block && (!Array.isArray(block))) { + let { start, end } = block; + block = [start, end]; + } + + // Our preferred properties are `line` and `block`, but if `start` and/or + // `end` are present, we can extract some value out of them. + + // If `start` and `end` both exist, they must identify block delimiters. + if (!block && meta.start && meta.end) { + block = [meta.start.trim(), meta.end.trim()]; + } + // If `start` exists but `end` does not, `start` must be a line delimiter. + if (!line && meta.start && !meta.end) { + line = meta.start.trim(); + } + + return { line, block }; +} + +// Convert comment delimiter metadata to the format expected by +// `LanguageMode::getCommentStringsForPosition`. We can act as a provider of +// this data if the traditional sources are empty. +function commentStringsFromDelimiters(meta) { + let { line, block } = normalizeDelimiters(meta); + let commentStartString; + let commentEndString; + let commentDelimiters = { line, block }; + let blockIsValid = block != null && Array.isArray(block); + let lineIsValid = typeof line === 'string'; + if (lineIsValid || blockIsValid) { + commentDelimiters = { line, block }; + if (lineIsValid) { + // The “Toggle Line Comment” command obviously prefers a line comment if + // one is present. + commentStartString = line; + } else if (blockIsValid) { + [commentStartString, commentEndString] = block; + } + } + let result = { commentStartString, commentEndString, commentDelimiters }; + return result; +} + + + +// Given a scope, return a single object of `editor.commentDelimiters` data. +// Needed because an ordinary config lookup will “blend” objects from cascading +// scopes — which is usually the behavior we want! Just not this time. +function getDelimitersForScope(scope) { + let reversed = [...scope.scopes].reverse(); + let mapped = reversed.map(scope => { + return atom.config.get('editor.commentDelimiters', { scope: [scope] }) + }) + let result = mapped.find(setting => !!setting) + return result ? normalizeDelimiters(result) : result +} + +module.exports = { + normalizeDelimiters, + commentStringsFromDelimiters, + getDelimitersForScope +}; diff --git a/src/text-editor.js b/src/text-editor.js index c464797d05..8132eb8411 100644 --- a/src/text-editor.js +++ b/src/text-editor.js @@ -5925,6 +5925,60 @@ module.exports = class TextEditor { } } + // Public: Return information about the appropriate comment delimiters to use + // at a given point in the buffer. + // + // Pulsar allows language bundles to define comment delimiters in several + // places. For instance, a grammar author can place delimiter metadata in the + // grammar definition file, or as scope-specific settings in the ordinary + // config system — or a combination of the two. + // + // In some languages, comment delimiters vary based on position in the + // buffer. (For instance, line comments can't always be used in JavaScript + // JSX blocks, so block comments are much safer.) This method will look for + // any such overrides and return what it thinks are the best delimiters to + // use at a given point. + // + // Some languages don't specify all their delimiters in their configuration, + // but this method will return all the information that it can discern. + // + // * point - A {Point} or point-compatible {Array}. + // + // Returns an {Object} with the following properties: + // + // * `line`: If present, a {String} representing a line comment delimiter. + // (If `undefined`, there is no known line comment delimiter for the given + // buffer position.) + // * `block`: If present, a two-item {Array} containing {String}s + // representing the starting and ending block comment delimiters. (If + // `undefined`, there are no known block comment delimiters for the given + // buffer position.) + // + getCommentDelimitersForBufferPosition(point) { + point = Point.fromObject(point); + const languageMode = this.buffer.getLanguageMode(); + let { + commentStartString, + commentEndString, + commentDelimiters + } = languageMode.commentStringsForPosition(point); + if (commentDelimiters) { + return commentDelimiters; + } else { + // Build a delimiters object out of the other data we received. The + // `commentStartString` and `commentEndString` settings aren't meant to + // be comprehensive — they just tell you which delimiter(s) to use to + // comment out a given selection — but they're better than nothing. + if (commentStartString && commentEndString) { + return { block: [commentStartString.trim(), commentEndString.trim()] }; + } else if (commentStartString && !commentEndString) { + return { line: commentStartString.trim() }; + } else { + return null; + } + } + } + rowRangeForParagraphAtBufferRow(bufferRow) { if (!NON_WHITESPACE_REGEXP.test(this.lineTextForBufferRow(bufferRow))) return; diff --git a/src/text-mate-language-mode.js b/src/text-mate-language-mode.js index aa1b52c9a2..f5092b157b 100644 --- a/src/text-mate-language-mode.js +++ b/src/text-mate-language-mode.js @@ -11,6 +11,7 @@ const { fromFirstMateScopeId } = require('./first-mate-helpers'); const { selectorMatchesAnyScope } = require('./selectors'); +const { normalizeDelimiters, commentStringsFromDelimiters } = require('./comment-utils.js'); const NON_WHITESPACE_REGEX = /\S/; @@ -29,7 +30,7 @@ class TextMateLanguageMode { this.id = params.id != null ? params.id : nextId++; this.buffer = params.buffer; this.largeFileMode = params.largeFileMode; - this.config = params.config; + this.config = params.config ?? atom.config; this.largeFileMode = params.largeFileMode != null ? params.largeFileMode @@ -236,10 +237,19 @@ class TextMateLanguageMode { const commentEndEntry = commentEndEntries.find(entry => { return entry.scopeSelector === commentStartEntry.scopeSelector; }); - return { - commentStartString: commentStartEntry && commentStartEntry.value, - commentEndString: commentEndEntry && commentEndEntry.value - }; + // If a `commentDelimiters` setting exists, return it in its entirety. This + // can contain more comprehensive delimiter metadata for snippets and other + // purposes. + const commentDelimiters = this.config.get('editor.commentDelimiters', { scope }); + if (commentStartEntry) { + return { + commentStartString: commentStartEntry && commentStartEntry.value, + commentEndString: commentEndEntry && commentEndEntry.value, + commentDelimiters: commentDelimiters && normalizeDelimiters(commentDelimiters) + }; + } else if (commentDelimiters) { + return commentStringsFromDelimiters(commentDelimiters); + } } /* diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 333b83d478..136dd82ae5 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -8,6 +8,7 @@ const TokenizedLine = require('./tokenized-line'); const TextMateLanguageMode = require('./text-mate-language-mode'); const { matcherForSelector } = require('./selectors'); const TreeIndenter = require('./tree-indenter'); +const { normalizeDelimiters, commentStringsFromDelimiters } = require('./comment-utils.js'); let nextId = 0; const MAX_RANGE = new Range(Point.ZERO, Point.INFINITY).freeze(); @@ -169,7 +170,16 @@ class TreeSitterLanguageMode { this.firstNonWhitespaceRange(position.row) || new Range(position, position); const { grammar } = this.getSyntaxNodeAndGrammarContainingRange(range); - return grammar.commentStrings; + const { grammar: originalPositionGrammar } = this.getSyntaxNodeAndGrammarContainingRange(new Range(position, position)); + + let result = commentStringsFromDelimiters(grammar.commentStrings); + console.log('result???', result, grammar.commentStrings); + if (originalPositionGrammar !== grammar) { + result.commentDelimiters = commentStringsFromDelimiters( + originalPositionGrammar.commentStrings + ).commentDelimiters; + } + return result; } isRowCommented(row) { @@ -1127,6 +1137,7 @@ class LayerHighlightIterator { } seek(targetIndex, containingTags, containingTagStartIndices) { + // eslint-disable-next-line no-empty while (this.treeCursor.gotoParent()) {} this.atEnd = true; diff --git a/src/wasm-tree-sitter-grammar.js b/src/wasm-tree-sitter-grammar.js index fc6c87647e..bed2fe8a6e 100644 --- a/src/wasm-tree-sitter-grammar.js +++ b/src/wasm-tree-sitter-grammar.js @@ -3,9 +3,11 @@ const path = require('path'); const Parser = require('./web-tree-sitter'); const { CompositeDisposable, Emitter } = require('event-kit'); const { File } = require('pathwatcher'); +const { normalizeDelimiters } = require('./comment-utils.js'); const parserInitPromise = Parser.init(); +// Extended: This class holds an instance of a Tree-sitter grammar. module.exports = class WASMTreeSitterGrammar { constructor(registry, grammarPath, params) { this.registry = registry; @@ -73,48 +75,42 @@ module.exports = class WASMTreeSitterGrammar { return id; } - // Retrieve the comment delimiters for this grammar. + // Extended: Retrieve all known comment delimiters for this grammar. + // + // Some grammars may have different delimiters for different parts of a file + // (such as JSX within JavaScript). In these cases, you might want to call + // {TextEditor::getCommentDelimitersForBufferPosition} with a `{Point}` in the + // buffer. + // + // Returns an {Object} with the following properties: + // + // * `line`: If present, a {String} representing a line comment delimiter. + // (If `undefined`, there is no known line comment delimiter for the given + // buffer position.) + // * `block`: If present, a two-item {Array} containing {String}s + // representing the starting and ending block comment delimiters. (If + // `undefined`, there are no known block comment delimiters for the given + // buffer position.) // - // Traditionally, grammars specified only the delimiters needed for the - // “Toggle Line Comments” command — either a line comment (if it existed) or - // a block comment. But other features might want to know _all_ of a - // language's possible comment delimiters, so we've devised new config values. getCommentDelimiters() { - let meta = this.commentMetadata; - if (!meta) { return null; } - - let result = {}; - - // The new convention is to specify a `line` property and start/end - // properties. - let { line, block } = this.commentMetadata; - - // Failing that, we can deliver at least a partial result by inspecting the - // older convention. If `start` exists but not `end`, we know `start` must - // be a line comment delimiter. - if (!line && meta.start && !meta.end) { - line = meta.start; - } - // Likewise, if both `start` and `end` exist, we know they must be block - // comment delimiters. - if (!block && meta.start && meta.end) { - block = { start: meta.start, end: meta.end }; - } + // Prefer the config system. It's a better place for this data to live. + let commentDelimiters = atom.config.get( + 'editor.commentDelimiters', + { scope: [this.scopeName] } + ); + if (commentDelimiters) return commentDelimiters; - // Strip all whitespace from delimiters. Whatever is consuming them can - // decide if it wants whitespace. - if (line) { - line = line.strip(); - result.line = line; + // Failing that, try to extract useful information from this metadata. + if (this.commentMetadata) { + return normalizeDelimiters(this.commentMetadata); } - if (block) { - block.start = block.start?.strip(); - block.end = block.end?.strip(); - result.block = block; - } + // If even that doesn't exist, we can fall back onto the older config + // settings. + let start = atom.config.get('editor.commentStart', { scope: [this.scope] }); + let end = atom.config.get('editor.commentEnd', { scope: [this.scope] }); - return result; + return normalizeDelimiters({ start, end }); } classNameForScopeId(id) { @@ -132,9 +128,12 @@ module.exports = class WASMTreeSitterGrammar { return this._language; } - // Returns the Tree-sitter language instance associated with this grammar - // once it loads. When the {Promise} returned by this method resolves, the - // grammar is ready to perform parsing and to execute query captures. + // Extended: Returns the Tree-sitter language instance associated with this + // grammar once it loads. + // + // Returns a {Promise} that will resolve with a Tree-sitter `Language` + // instance. Once it resolves, the grammar is ready to perform parsing and to + // execute query captures. async getLanguage() { await parserInitPromise; if (!this._language) { @@ -244,11 +243,19 @@ module.exports = class WASMTreeSitterGrammar { return query; } - // Async, but designed so that multiple near-simultaneous calls to `getQuery` - // from multiple buffers will not cause multiple calls to `language.query`, - // since it's a major bottleneck. Instead they all receive the same unsettled - // promise. + // Extended: Given a kind of query, retrieves a Tree-sitter `Query` object + // in async fashion. + // + // * `queryType` A {String} describing the query type: typically one of + // `highlightsQuery`, `foldsQuery`, `tagsQuery`, or `indentsQuery`, + // but could be any other custom type. + // + // Returns a {Promise} that resolves to a Tree-sitter `Query` object. getQuery(queryType) { + // Async, but designed so that multiple near-simultaneous calls to + // `getQuery` from multiple buffers will not cause multiple calls to + // `language.query`, since it's a major bottleneck. Instead they all + // receive the same unsettled promise. // let inDevMode = atom.inDevMode(); let query = this.queryCache.get(queryType); if (query) { return Promise.resolve(query); } @@ -279,18 +286,28 @@ module.exports = class WASMTreeSitterGrammar { return promise; } - // Creates an arbitrary query from this grammar. Package authors and end - // users can use queries for whatever purposes they like. + // Extended: Creates an arbitrary query from this grammar. Package authors + // and end users can use queries for whatever purposes they like. + // + // * `queryContents` A {String} representing the entire contents of a query + // file. Can contain any number of queries. + // + // Returns a {Promise} that will resolve to a Tree-sitter `Query` object. async createQuery(queryContents) { let language = await this.getLanguage(); return language.query(queryContents); } - // Creates an arbitrary query from this grammar. Package authors and end - // users can use queries for whatever purposes they like. + // Extended: Creates an arbitrary query from this grammar. Package authors + // and end users can use queries for whatever purposes they like. // // Synchronous; use only when you can be certain that the tree-sitter // language has already loaded. + // + // * `queryContents` A {String} representing the entire contents of a query + // file. Can contain any number of queries. + // + // Returns a Tree-sitter `Query` object. createQuerySync(queryContents) { if (!this._language) { throw new Error(`Language not loaded!`); @@ -336,14 +353,16 @@ module.exports = class WASMTreeSitterGrammar { } } - // Calls `callback` when any of this grammar's query files change. + // Extended: Calls `callback` when any of this grammar's query files change. // // The callback is invoked with an object argument with two keys: // - // - `filePath`: The path to the query file on disk. - // - `queryType`: The type of query file, as denoted by its configuration key - // in the grammar file. One of `highlightsQuery`, `indentsQuery`, - // `foldsQuery`, or `localsQuery`. + // * `callback`: The callback to be invoked. Takes one argument: + // * `data`: An object with keys: + // * `filePath`: The path to the query file on disk. + // * `queryType`: The type of query file, as denoted by its + // configuration key in the grammar file. Usually one of + // `highlightsQuery`, `indentsQuery`, `foldsQuery`, or `tagsQuery`. onDidChangeQueryFile(callback) { return this.emitter.on('did-change-query-file', callback); } @@ -360,8 +379,8 @@ module.exports = class WASMTreeSitterGrammar { this.queryCache.clear(); } - // Define a set of rules for when this grammar should delegate to a different - // grammar for certain regions of a buffer. Examples: + // Extended: Define a set of rules for when this grammar should delegate to a + // different grammar for certain regions of a buffer. Examples: // // * embedding one language inside another (e.g., JavaScript in HTML) // * tokenizing certain structures with greater detail (e.g., regular @@ -370,56 +389,58 @@ module.exports = class WASMTreeSitterGrammar { // comments in JavaScript) // // This differs from TextMate-style injections, which operate at the scope - // level and are currently incompatible with tree-sitter grammars. - // - // Expects an object with these keys: - // - // * `type` (string): The type of node to inject into. - // * `language` (function): Should return a string describing the language - // that should inject into this area. Grammars that can inject into others - // will define an `injectionRegex` property that will be tested against - // this value; the longest match will win. - // The function receives the node itself as an argument, so you can decide - // the language based on the content of the node, or return `undefined` if - // an injection should not take place. - // * `content` (function): Receives the matching node and should return the - // node that will actually be injected into. Usually this will be the same - // node that was given, but could also be a specific child or descendant of - // that node, or potentially any other node in the tree. + // level and are currently incompatible with Tree-sitter grammars. // - // Understands the following optional keys: + // NOTE: Packages will call {::addInjectionPoint} with a given scope name, + // and that call will be delegated to any Tree-sitter grammars that match + // that scope name, whether they're legacy Tree-sitter or modern Tree-sitter. + // But modern Tree-sitter grammars cannot be injected into legacy Tree-sitter + // grammars, and vice versa. // - // * `includeChildren` (boolean): Whether the injection range should include - // the ranges of this node's children. Defaults to `false`, meaning that - // the range of each of this node's children will be "subtracted" from the - // injection range, and the remainder will be parsed as if those ranges of - // the buffer do not exist. - // * `includeAdjacentWhitespace` (boolean): Whether the injection range - // should include whitespace that occurs between content nodes. Defaults to - // `false`. When `true`, if two injection ranges are separated from one - // another by only whitespace, that whitespace will be added to the - // injection range, and the ranges will be consolidated. - // * `newlinesBetween` (boolean): Whether the injection range should include - // any newline characters that may exist in between injection ranges. - // Defaults to `false`. Grammars like ERB and EJS need this so that they do - // not interpret two different embedded code sections on different lines as - // occurring on the same line. - // * `coverShallowerScopes` (boolean): Whether the injection should prevent - // the parent grammar (and any of its ancestors) from applying scope - // boundaries within its injection range(s). Defalts to `false`. - // * `languageScope` (string | function | null): The base language scope that - // should be used by this injection. Defaults to the grammar's own - // `scopeName` property. Set this to a string to override the default scope - // name, or `null` to omit a base scope name altogether. Set this to a - // function if the scope name to be applied varies based on the grammar; - // the function will be called with a grammar instance as its only - // argument. + // * `options` The options for the injection point: + // * `type` A {String} describing type of node to inject into. + // * `language` A {Function} that should return a string describing the + // language that should inject into this area. The string should be a + // short, unambiguous description of the language; it will be tested + // against other grammars’ `injectionRegex` properties. Receives one + // parameter: + // * `node` A Tree-sitter node. + // * `content` A {Function} that should return the node (or nodes) that + // will actually be injected into. Usually this will be the same node + // that was given, but could also be a specific child or descendant of + // that node. + // * `includeChildren` (optional) {Boolean} controlling whether the + // injection range should include the ranges of the content node’s + // children. Defaults to `false`, meaning that the range of each of this + // node's children will be "subtracted" from the injection range, and the + // remainder will be parsed as if those ranges of the buffer do not + // exist. + // * `includeAdjacentWhitespace` (optional) {Boolean} controlling whether + // the injection range should include whitespace that occurs between + // content nodes. Defaults to `false`. When `true`, if two injection + // ranges are separated from one another by only whitespace, that + // whitespace will be added to the injection range, and the ranges will + // be consolidated. + // * `newlinesBetween` (optional) {Boolean} controlling whether the + // injection range should include any newline characters that may exist + // in between injection ranges. Defaults to `false`. Grammars like ERB + // and EJS need this so that they do not interpret two different + // embedded code sections on different lines as occurring on the same + // line. + // * `coverShallowerScopes` (optional) {Boolean} controlling whether the + // injection should prevent the parent grammar (and any of its + // ancestors) from applying scope boundaries within its injection + // range(s). Defalts to `false`. + // * `languageScope` (optional) A value that determines what scope, if + // any, is added to the injection as its “base” scope name. Can be a + // {String}, {null}, or a {Function} that returns either of these values. + // The base language scope that should be used by this injection. + // Defaults to the grammar's own `scopeName` property. Set this to a + // string to override the default scope name, or `null` to omit a base + // scope name altogether. Set this to a function if the scope name to be + // applied varies based on the grammar; the function will be called with + // a grammar instance as its only argument. // - // NOTE: Packages will call `atom.grammars.addInjectionPoint` with a given - // scope name, and that call will be delegated to any tree-sitter grammars - // that match that scope name, whether they're legacy-tree-sitter or - // modern-tree-sitter. But modern-tree-sitter grammars cannot be injected - // into by legacy-tree-sitter-grammars, and vice versa. // addInjectionPoint(injectionPoint) { let { type } = injectionPoint; diff --git a/src/wasm-tree-sitter-language-mode.js b/src/wasm-tree-sitter-language-mode.js index 18ead35aae..922382d4ff 100644 --- a/src/wasm-tree-sitter-language-mode.js +++ b/src/wasm-tree-sitter-language-mode.js @@ -8,6 +8,7 @@ const ScopeResolver = require('./scope-resolver'); const Token = require('./token'); const TokenizedLine = require('./tokenized-line'); const { matcherForSelector } = require('./selectors'); +const { commentStringsFromDelimiters, getDelimitersForScope } = require('./comment-utils.js'); const createTree = require('./rb-tree'); @@ -154,7 +155,7 @@ class WASMTreeSitterLanguageMode { this.id = nextLanguageModeId++; this.buffer = buffer; this.grammar = grammar; - this.config = config; + this.config = config ?? atom.config; this.grammarRegistry = grammars; this.syncTimeoutMicros = syncTimeoutMicros ?? PARSE_JOB_LIMIT_MICROS; @@ -797,8 +798,8 @@ class WASMTreeSitterLanguageMode { getSyntaxNodeAndGrammarContainingRange(range, where = FUNCTION_TRUE) { if (!this.rootLanguageLayer) { return { node: null, grammar: null }; } - let layersAtStart = this.languageLayersAtPoint(range.start); - let layersAtEnd = this.languageLayersAtPoint(range.end); + let layersAtStart = this.languageLayersAtPoint(range.start, { exact: true }); + let layersAtEnd = this.languageLayersAtPoint(range.end, { exact: true }); let sharedLayers = layersAtStart.filter( layer => layersAtEnd.includes(layer) ); @@ -814,16 +815,17 @@ class WASMTreeSitterLanguageMode { let rootNode = layer.tree.rootNode; if (!rootNode.range.containsRange(range)) { - if (layer === this.rootLanguageLayer) { - // This layer is responsible for the entire buffer, but our tree's - // root node may not actually span that entire range. If the buffer - // starts with empty lines, the tree may not start parsing until the - // first non-whitespace character. - // - // But this is the root language layer, so we're going to pretend - // that our tree's root node spans the entire buffer range. - results.push({ node: rootNode, grammar, depth }); - } + // There's often a difference between (a) the areas that we consider to + // be our canonical content ranges for a layer and (b) the range + // covered by the layer's root node. Root tree nodes usually ignore any + // whitespace that occurs before the first meaningful content of the + // node, but we consider that space to be under the purview of the + // layer all the same. + // + // If we've gotten this far, we've already decided that this layer + // includes this range. So let's just pretend that the root node covers + // this area. + results.push({ node: rootNode, grammar, depth }); continue; } @@ -899,17 +901,18 @@ class WASMTreeSitterLanguageMode { let { depth, grammar } = layer; let rootNode = layer.tree.rootNode; if (!rootNode.range.containsPoint(position)) { - if (layer === this.rootLanguageLayer) { - // This layer is responsible for the entire buffer, but our tree's - // root node may not actually span that entire range. If the buffer - // starts with empty lines, the tree may not start parsing until the - // first non-whitespace character. - // - // But this is the root language layer, so we're going to pretend - // that our tree's root node spans the entire buffer range. - if (where(rootNode, grammar)) { - results.push({ rootNode: node, depth }); - } + // There's often a difference between (a) the areas that we consider to + // be our canonical content ranges for a layer and (b) the range + // covered by the layer's root node. Root tree nodes usually ignore any + // whitespace that occurs before the first meaningful content of the + // node, but we consider that space to be under the purview of the + // layer all the same. + // + // If we've gotten this far, we've already decided that this layer + // includes this point. So let's just pretend that the root node covers + // this area. + if (where(rootNode, grammar)) { + results.push({ rootNode: node, depth }); } continue; } @@ -1087,12 +1090,33 @@ class WASMTreeSitterLanguageMode { // scope-specific setting for scenarios where a language has different // comment delimiters for different contexts. // - // TODO: Our understanding of the correct delimiters for a given buffer - // position is only as granular as the entire buffer row. This can bite us in - // edge cases like JSX. It's the right decision if the user toggles a comment - // with an empty selection, but if specific buffer text is selected, we - // should look up the right delmiters for that specific range. This will - // require a new branch in the “Editor: Toggle Line Comments” command. + // Returns `commentStartString` and (sometimes) `commentEndString` + // properties. If only the former is a {String}, then “Toggle Line Comments” + // will insert a line comment; if both are {String}s, it'll insert a block + // comment. + // + // NOTE: This method also returns a `commentDelimiters` property with + // metadata about the comment delimiters at the given position. Since the + // main purpose of this method, historically, has been to determine which + // delimiter(s) to use for the “Toggle Line Comment” command, we adjust the + // position we're given to cover the first non-whitespace content on the line + // for more accurate results. But `commentDelimiters` contains unadjusted + // data wherever possible because we don't make assumptions about how the + // caller will use the data. + // + // This might produce surprising results sometimes — like `commentDelimiters` + // containing delimiters from a different language than the delimiters in the + // other returned properties. But that's OK. Consumers of this function will + // know why those properties disagree and which one they're most interested + // in, and it still makes sense for these different use cases to share code. + // + // TODO: When toggling comments on a line or buffer range, our understanding + // of the correct delimiters for a given buffer position is only as granular + // as the entire buffer row. This can bite us in edge cases like JSX. It's + // the right decision if the user toggles a comment with an empty selection, + // but if specific buffer text is selected, we should look up the right + // delimiters for that specific range. This will require a new branch in the + // “Editor: Toggle Line Comments” command. commentStringsForPosition(position) { const range = this.firstNonWhitespaceRange(position.row) || new Range(position, position); @@ -1105,28 +1129,72 @@ class WASMTreeSitterLanguageMode { const commentEndEntries = this.config.getAll( 'editor.commentEnd', { scope }); - const commentStartEntry = commentStartEntries[0]; - const commentEndEntry = commentEndEntries.find(entry => ( - entry.scopeSelector === commentStartEntry.scopeSelector - )); + // If a `commentDelimiters` setting exists, attach it to the return object. + // This can contain more comprehensive delimiter metadata for snippets and + // other purposes. + // + // This is just general metadata. We don't know the user's intended use + // case. So we should look up the scope descriptor of the _original_ + // position, not the one at the beginning of the line. + const originalScope = this.scopeDescriptorForPosition(position); + const commentDelimiters = getDelimitersForScope(originalScope); + + // The two config entries are separate, but if they're paired, then we need + // to make sure we're reading them from the same layer. Otherwise we could + // wind up with, say, `