[#6 - #19 - #20] - tokenizer: improve algorithm by using a node-based…

… approach instead of the token jumper (#21)

- algorithm which detects declarations even better due to the newly implemented node scanning logic
- multiple node detections make the implementation of an autofixer possible

docs/rules/var-spacing.md

@@ -1,6 +1,8 @@
 var-spacing
 ===========
 
+**Fixable:** This rule is automatically fixable using the `--fix` flag on the command line.
+
 ### Introduction
 
 This rules validates the correct alignment of variable declarations:

lib/rules/var-spacing.rule.js

@@ -12,44 +12,45 @@
 
 var validatePunctuators = require('../util/validatePunctuators');
 var findNextPunctuators = require('../util/findNextPunctuators');
-var findFirstVarAssignment = require('../util/findFirstVarAssignment');
 
 module.exports = function (context) {
-  var source = context.getSourceCode(), sourceLines = context.getSourceLines();
+  var source = context.getSourceCode();
 
   function declarationCheck(node) {
-    // skip if the variable is declared, but not assigned to a value
-    var first = findFirstVarAssignment(node);
-    if (first !== null) {
-      // parse through the next AST tokens in order
-      // to gather the next relevant tokens being necessary
-      // for the alignment validation.
-      validatePunctuators(
-        source.getTokensBetween(first.id, first.init).shift(),
-        findNextPunctuators(node, source, sourceLines),
-        context,
-        node
-      );
+    if (-1 === ['Program', 'BlockStatement'].indexOf(node.parent.type)) {
+      return;
     }
+    validatePunctuators(
+      findNextPunctuators(node, source),
+      context,
+      node,
+      source
+    );
   }
 
   function expressionCheck(node) {
+    function checkOperator(node) {
+      var allowedOperators = ['=', '+=', '-=', '*=', '/=', '%=', '<<=', '>>=', '>>>=', '|=', '^=', '&='];
+      return -1 === allowedOperators.indexOf(node.operator)
+        || -1 === ['Program', 'BlockStatement', 'ExpressionStatement', 'SequenceExpression'].indexOf(node.parent.type);
+    }
+
     // only check for member expressions with an equal sign.
     // expressions like these `foo: bar` are handled by `key-spacing` and don't need to be validated.
-    if (node.parent.operator !== '=' || node.parent.parent.type === 'AssignmentExpression') {
+    if (checkOperator(node)) {
       return;
     }
     validatePunctuators(
-      source.getTokensBetween(node.parent.left, node.parent.right).shift(),
-      findNextPunctuators(node, source, sourceLines, node),
+      findNextPunctuators(node, source),
       context,
-      node
+      node,
+      source
     );
   }
 
   return {
-    'VariableDeclaration': declarationCheck,
-    'MemberExpression':    expressionCheck
+    'VariableDeclaration':  declarationCheck,
+    'AssignmentExpression': expressionCheck
   };
 };
 

lib/util/columnComputation.js

@@ -0,0 +1,41 @@
+/*
+ * This file is part of the Sententiaregum project.
+ *
+ * (c) Maximilian Bosch <[email protected]>
+ * (c) Ben Bieler <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+'use strict';
+
+function bigger(a, b) {
+  return a > b ? a : b;
+}
+
+module.exports = {
+  computeAppropriateColumn: function (items, context, wrongColumnHandler) {
+    var source = context.getSourceCode();
+    var result = items.reduce(function (prev, cur) {
+      var curCol = cur.loc.start.column;
+      if (Math.abs(source.getTokenBefore(cur).loc.end.column - curCol) > 1) {
+        return prev;
+      }
+      return bigger(prev, curCol);
+    }, null);
+
+    if (null === result) {
+      wrongColumnHandler();
+      return;
+    }
+
+    return result;
+  },
+  computeActualColumn: function (items, source) {
+    return items.reduce(function (prev, cur) {
+      var col = source.getTokenBefore(cur).loc.end.column + 1;
+      return bigger(prev, col);
+    }, 0);
+  }
+};

lib/util/findNextPunctuators.js

@@ -10,66 +10,77 @@
 
 'use strict';
 
-var jumpAssignmentExpressions = require('./jumpAssignmentExpressions');
+var registerToken = require('./registerToken');
 
-module.exports = function(node, source, lines) {
-  function isVariableAssignment(token) {
-    return typeof token.declarations !== 'undefined' && typeof token.computed === 'undefined';
+module.exports = function(node, source) {
+  if (registerToken.validateDeclarations(node)) {
+    return [];
   }
 
-  function isPunctuatorAssignment(token) {
-    return token.type === 'Punctuator' && token.value === '=';
-  }
+  var exprNode = getExpressionNode(node),
+      blockStmt = exprNode.parent,
+      tokens = blockStmt.body,
+      index = tokens.indexOf(exprNode),
+      statements = parseStatements(tokens.slice(0).splice(index, tokens.length - index));
 
-  function getPunctuators(node, source) {
-    function isMultipleLineDeclaration(node, token) {
-      return node.loc.start.line < token.loc.start.line;
-    }
+  return gatherAssignmentsFromNodes(statements, source);
+};
+
+function gatherAssignmentsFromNodes(statements, source) {
+  return statements.reduce(function (list, stmt) {
+    registerToken.register(stmt);
+    switch (stmt.type) {
+      case 'ExpressionStatement':
+      case 'AssignmentExpression':
+        var expr = stmt.type === 'ExpressionStatement' ? stmt.expression : stmt,
+            expressions = expr.type === 'SequenceExpression' ? expr.expressions : [expr];
 
-    function gatherPunctuatorsFromCurrentDeclaration(node, source) {
-      return node.declarations.reduce(function (tokens, subToken, index) {
-        // skip first one and empty declarations
-        if (subToken.init && index > 0) {
-          var punctuator = source.getTokensBetween(subToken.id, subToken.init).shift();
-          if (isPunctuatorAssignment(punctuator) && isMultipleLineDeclaration(node, punctuator)) {
-            tokens.push(punctuator);
+        return list.concat(expressions.reduce(function (list, token, i) {
+          var prev = expressions[i - 1];
+          if (token.right && (typeof prev === 'undefined' || token.loc.start.line - prev.loc.end.line === 1)) {
+            list.push(source.getTokensBetween(token.left, token.right).shift());
           }
-        }
-        return tokens;
-      }, []);
+          return list;
+        }, []));
+      case 'VariableDeclaration':
+        var punctuators = [];
+        stmt.declarations.forEach(function (declaration) {
+          if (checkDeclaration(declaration, punctuators)) {
+            punctuators.push(source.getTokensBetween(declaration.id, declaration.init).shift());
+          }
+        });
+
+        return list.concat(punctuators);
+      default:
+        return list;
     }
+  }, []);
+}
 
-    var punctuators = [];
-    if (isVariableAssignment(node) && node.declarations.length > 1) {
-      punctuators = gatherPunctuatorsFromCurrentDeclaration(node, source);
+function parseStatements(statements) {
+  return statements.reduce(function (list, stmt) {
+    var previous = list[list.length - 1], start = stmt.loc.start.line;
+    if (checkToken(stmt, previous, start)) {
+      list.push(stmt);
     }
+    return list;
+  }, []);
+}
 
-    return punctuators;
-  }
+function checkDeclaration(declaration, punctuators) {
+  return declaration.init
+    && (punctuators.length === 0 || punctuators[punctuators.length - 1].loc.start.line !== declaration.loc.start.line);
+}
 
-  function getNextExpression(source, node) {
-    function findNextExpressionToken(source, node) {
-      var isVarAssignment = isVariableAssignment(node), amount = !isVarAssignment ? 2 : 1;
-      return source.getTokenAfter(!isVarAssignment ? node.parent : node, amount);
-    }
 
-    var next = findNextExpressionToken(source, node);
-    if (typeof next === 'undefined') {
-      throw 'Failed to receive next expression!';
-    }
-    return jumpAssignmentExpressions(next, source);
-  }
+function checkToken(stmt, previous, start) {
+  return -1 !== ['VariableDeclaration', 'AssignmentExpression', 'ExpressionStatement'].indexOf(stmt.type)
+    && (typeof previous === 'undefined' || (start - previous.loc.end.line) === 1);
+}
 
-  function getDeclarations(node, source, lines) {
-    try {
-      var nextLine = lines[node.loc.end.line], punctuators = getPunctuators(node, source), punctuator;
-      if (typeof nextLine !== 'undefined' && nextLine.trim().length !== 0) {
-        punctuator = getNextExpression(source, node);
-      }
-    } finally {
-      return punctuator ? punctuators.concat([punctuator]) : punctuators;
-    }
+function getExpressionNode(node) {
+  if ('AssignmentExpression' === node.type) {
+    return node.parent.type === 'SequenceExpression' ? node.parent.parent : node.parent;
   }
-
-  return getDeclarations(node, source, lines);
-};
+  return node;
+}

lib/util/fixer.js

@@ -0,0 +1,55 @@
+/*
+ * This file is part of the Sententiaregum project.
+ *
+ * (c) Maximilian Bosch <[email protected]>
+ * (c) Ben Bieler <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+'use strict';
+
+var columnComputation = require('./columnComputation');
+
+function repeat(val, times) {
+  var buffer = '';
+  for (var i = 0; i < times; i++) {
+    buffer += val;
+  }
+
+  return buffer;
+}
+
+function removeAlignmentRange(source, punctuator, fixer, column) {
+  var before = source.getTokenBefore(punctuator);
+  return fixer.removeRange([
+    before.range[1] + (column - before.loc.end.column),
+    punctuator.range[0]
+  ]);
+}
+
+module.exports = {
+  getWrongAlignmentFixer: function (column, cur, punctuator, source) {
+    return function(fixer) {
+      if (column > cur) {
+        return fixer.insertTextBefore(punctuator, repeat(' ', column - cur));
+      }
+      return removeAlignmentRange(source, punctuator, fixer, column);
+    };
+  },
+  getWrongColumnHandler: function(items, source, context) {
+    return function () {
+      var actual = columnComputation.computeActualColumn(items, source);
+      items.forEach(function (token) {
+        context.report({
+          message: 'The punctuator column must be placed directly after the longest variable!',
+          node:    token,
+          fix:     function (fixer) {
+            return removeAlignmentRange(context.getSourceCode(), token, fixer, actual);
+          }
+        });
+      });
+    }
+  }
+};