diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..0ffbac4 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,6 @@ +{ + "extends": "eslint-config-fluid", + "env": { + "node": true + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..432d0cd --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules +package-lock.json +coverage +reports +.vagrant diff --git a/.nycrc b/.nycrc new file mode 100644 index 0000000..b11d4e0 --- /dev/null +++ b/.nycrc @@ -0,0 +1,6 @@ +{ + "reporter": ["html", "text-summary"], + "report-dir": "reports", + "temp-directory": "coverage", + "include": ["src/**/*.js"] +} diff --git a/.vagrant.yml b/.vagrant.yml new file mode 100644 index 0000000..0dfe7af --- /dev/null +++ b/.vagrant.yml @@ -0,0 +1,25 @@ +# QI Configuration file, see: https://github.com/amatas/vagrant-gpii-ci +--- + +env: + vms: + windows10: + cpu: 2 # number of cpus + memory: 2048 # amount of RAM memory (in Mb) + clone: true # use the linked_clone Vagrant feature + autostart: true + box: inclusivedesign/windows10-eval-x64-Apps + +stages: # Stages to perform when 'ci test' command is invoked + - setup # Install our system-level dependencies, etc. + - test # Run the actual tests + +setup_job: + stage: setup + script: + - choco install nodejs-lts -y + +test_job: + stage: test # name of the stage + script: # One line per command to execute + - "do.ps1 -c 'v: && npm install && npm test'" diff --git a/Gruntfile.js b/Gruntfile.js new file mode 100644 index 0000000..d341149 --- /dev/null +++ b/Gruntfile.js @@ -0,0 +1,32 @@ +/* eslint-env node */ +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); +require("./"); + +module.exports = function (grunt) { + var globbedSources = { + md: ["./*.md"], + js: ["./*.js", "./src/**/*.js", "./tests/**/*.js", "./tests/*.js", "./*.js"], + json: ["./*.json", "!./package-lock.json"], + json5: [], + other: ["./.*"] + }; + + // We manually resolve our globs to raw paths to ensure that our code is used rather than + // the copy of ourselves we inherit from gpii-grunt-lint-all. In regular usage, you should + // simply pass the globs themselves. + var fullPathSources = fluid.transform(globbedSources, function (globbedPaths) { + return gpii.glob.findFiles("%gpii-glob", globbedPaths, [], {dot: true}); + }); + + grunt.initConfig({ + lintAll: { + sources: fullPathSources, + expandPaths: false // This package is really the only one that should use this option. + } + }); + + grunt.loadNpmTasks("gpii-grunt-lint-all"); + grunt.registerTask("lint", "Perform all standard lint checks.", ["lint-all"]); +}; diff --git a/README.md b/README.md index 783d88a..8cee5e7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,160 @@ # gpii-glob -GPII Glob is for.... + +This library provides a means of determining a list of relevant files from a given location based on one or more +"globbed" patterns. Its API consists of a single static function (see below). + +This package was written as a drop-in replacement for grunt's globbing, which can be quite inefficient when working with +packages that have a lot of dependencies or other content that you wish to exclude. In short, grunt's strategy appears +to be to perform a full recursive scan of the directory before deciding whether to include or exclude files. By +contrast, this package only recursively scans content that potentially matches at least one "include". In real-world +testing, this strategy seems to result in linting runs that take around a tenth of the time. + +To achieve this, this package disallows overly broad patterns like '**/*.js' and some of the more advanced features of +the underlying library, such as regular expressions. See below for full details. + +## `gpii.glob.findFiles(rootPath, includes, [excludes], [minimatchOptions])` + +* `rootPath`: A full or package-relative directory to scan for matching files. +* `includes`: An `Array` of glob patterns that should be included in the results. +* `excludes`: An optional `Array` of glob patterns that should be excluded from the results. +* `minimatchOptions`: An optional `Object` detailing configuration options to be passed to [minimatch](https://github.com/isaacs/minimatch#options). +* Returns: An `Array` of full paths to files that match the supplied glob patterns. + +## "glob" Patterns + +A "glob" pattern is a string that describes the path to one or more files. It may contain single-asterisk wildcards +such as `*.js`. Single asterisks are only matched within a single path segment. So, for example, `./test/*-node*.js` +matches `./test/all-node-tests.js`, but not `./test/js/another-node-test.js`. + +"Glob" patterns may also use double-asterisks to indicate that any number of subdirectories may appear between one part +of a pattern and the next. So, for example, `./src/**/*.js` matches `./src/index.js` as well as +`./src/js/other.js`. + +The underlying concept of a glob is powerful, but can lead to inefficient lookup strategies. For the sake of +performance, the following are not allowed in patterns used with this library: + +1. Patterns starting with `./**` or `**`, which might require traversing all subdirectories before excludes can be + applied. +2. Patterns that attempt to break out of the starting directory, i.e. that start with `../`. +3. Patterns that use regular expressions to represent one or more parts of the path. +4. Patterns that use the windows backslash separator in any part of the path. + +Patterns can be negated by prepending an exclamation point. This mechanism allows you to define a more general rule and +then identify one or more exceptions to that rule. See below for examples. + +## Usage Examples + +Let's say you have a package called "my-package" whose structure looks roughly as diagrammed in this list: + +* (repository root) + * README.md + * index.js + * package.json + * .gitignore + * .eslintrc.json + * src + * lib + * forked-deps.js + * js + * index.js + * tests + * all-tests.js + * js + * test1.js + * test2.js + +Let's start by demonstrating includes. Content can only be brought into scope by a regular (non-negated) include. + +```javascript +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); + +require("gpii-glob"); + +// Let's assume that `fluid.module.resolvePath("%my-package")` resolves to `/source/my-package` for the purposes of +// these examples. +fluid.require("%my-package"); + +gpii.glob.findFiles("%my-package", [], [], {}); +// Returns: An empty array, as there are no includes. + +gpii.glob.findFiles("%my-package", ["./src/**/*.js"], [], {}); +// Returns: ["/source/my-package/src/js/index.js", "/source/my-package/src/lib/forked-deps.js"] +``` + +Please note, in order to use the package-relative notation as show above, you must register your package using +[`fluid.module.register`](https://docs.fluidproject.org/infusion/development/NodeAPI.html#fluidmoduleregistername-basedir-modulerequire) +and either `require` or [`fluid.require`](https://docs.fluidproject.org/infusion/development/NodeAPI.html#fluidrequiremodulename-foreignrequire-namespace) +your package. + +Negated includes and excludes take precedence over includes, i.e. they remove material from the results: + +```javascript +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); + +require("gpii-glob"); + +// Let's assume that `fluid.module.resolvePath("%my-package")` resolves to `/source/my-package` for the purposes of +// these examples. +fluid.require("%my-package"); + +gpii.glob.findFiles("%my-package", ["./src/**/*.js", "!./src/lib/**/*.js"], [], {}); +// Returns: ["/source/my-package/src/js/index.js"] + +// A negated include is basically the same as an exclude. +gpii.glob.findFiles("%my-package", ["./src/**/*.js"], ["./src/lib/**/*.js"], {}); +// Also returns: ["/source/my-package/src/js/index.js"] +``` + +A negated exclude takes precedence over both negated includes and regular excludes. + +```javascript +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); + +require("gpii-glob"); + +// Let's assume that `fluid.module.resolvePath("%my-package")` resolves to `/source/my-package` for the purposes of +// these examples. +fluid.require("%my-package"); + +// A negated exclude takes precedence over both negated includes and regular excludes. +gpii.glob.findFiles("%my-package", ["./tests/**/*.js"], ["./tests/js/**/*.js", "!./tests/js/test1.js"], {}); +// Returns: [ +// "/source/my-package/tests/all-tests.js", +// "/source/my-package/tests/js/test1.js", +// ] +``` + +The file `test1.js` is brought back into the list of matching files by the negated exclude, `test2.js` remains excluded. + +By default, filename wildcards (such as `*.json`) do not explicitly match "dot files". By passing the relevant option +to the underlying "minimatch" library, you can change this behaviour as shown here. + +```javascript +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); + +require("gpii-glob"); + +// Let's assume that `fluid.module.resolvePath("%my-package")` resolves to `/source/my-package` for the purposes of +// these examples. +fluid.require("%my-package"); + +// A filename wildcard search with the default minimatch options. +gpii.glob.findFiles("%my-package", ["./*.json"], [], {}); +// Returns: ["/source/my-package/package.json"] + +// A filename wildcard search with custom minimatch options. +gpii.glob.findFiles("%my-package", ["./*.json"], [], { dot: true }); +// Returns: ["/source/my-package/.eslintrc.json", "/source/my-package/package.json"] +``` + +For a full list of minimatch options, see [their documentation](https://github.com/isaacs/minimatch#options). Please +note, minimatch options only control which files match. This package uses its own means of evaluating whether a +directory *might* contain matching content, and minimatch options will not affect this. diff --git a/index.js b/index.js new file mode 100644 index 0000000..85aae9a --- /dev/null +++ b/index.js @@ -0,0 +1,6 @@ +/* eslint-env node */ +"use strict"; +var fluid = require("infusion"); +fluid.module.register("gpii-glob", __dirname, require); + +require("./src/js/glob.js"); diff --git a/package.json b/package.json new file mode 100644 index 0000000..f749781 --- /dev/null +++ b/package.json @@ -0,0 +1,28 @@ +{ + "name": "gpii-glob", + "version": "1.0.0", + "description": "A library to standardise resolving \"globbed\" file patterns within GPII projects.", + "main": "index.js", + "scripts": { + "pretest": "node node_modules/rimraf/bin.js coverage/* reports/*", + "test": "node node_modules/nyc/bin/nyc.js node tests/all-tests.js" + }, + "repository": "https://github.com/GPII/gpii-glob.git", + "author": "Tony Atkins ", + "license": "BSD-3-Clause", + "homepage": "https://github.com/GPII/gpii-glob#readme", + "dependencies": { + "infusion": "3.0.0-dev.20180801T212157Z.09bf3d438", + "minimatch": "3.0.4" + }, + "devDependencies": { + "eslint": "5.1.0", + "eslint-config-fluid": "1.3.0", + "gpii-grunt-lint-all": "1.0.1-dev.20180706T153657Z.4cbbd61", + "grunt": "1.0.3", + "mkdirp": "0.5.1", + "node-jqunit": "1.1.8", + "nyc": "13.0.1", + "rimraf": "2.6.2" + } +} diff --git a/src/js/glob.js b/src/js/glob.js new file mode 100644 index 0000000..7eee929 --- /dev/null +++ b/src/js/glob.js @@ -0,0 +1,405 @@ +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); + +var path = require("path"); +var fs = require("fs"); +var minimatch = require("minimatch"); + +fluid.registerNamespace("gpii.glob"); + +/** + * + * Find all files beneath a root directory based on a list of includes and excludes. Includes and excludes can be + * full, package-relative, or "glob" paths, see the README for examples. All paths are "pathed", i.e. resolved relative + * to `rootPath`, and then passed to `gpii.glob.scanSingleDir` to begin a recursive scan (see those docs for more + * details). + * + * @param {String} rootPath - A full or package-relative path to search. + * @param {Array} includes - An array of full or package-relative paths to include in the search results. + * @param {Array} excludes - An array of full or package-relative paths to exclude from the search results. + * @param {Object} [minimatchOptions] - (Optional) options to pass to minimatch. + * @param {Object|Array} [rules] - An optional set of custom rules defining invalid patterns as regular expressions. + * @return {Array} - An array of full paths to all matching files. + * + */ +gpii.glob.findFiles = function (rootPath, includes, excludes, minimatchOptions, rules) { + var invalidIncludes = gpii.glob.validatePatternArray(includes, rules); + var invalidExcludes = gpii.glob.validatePatternArray(excludes, rules); + if (invalidIncludes.length || invalidExcludes.length) { + if (invalidIncludes.length) { + gpii.glob.logInvalidRuleFeedback(invalidIncludes); + } + if (invalidExcludes.length) { + gpii.glob.logInvalidRuleFeedback(invalidExcludes); + } + + fluid.fail("One or more glob patterns you have entered are invalid. Cannot continue."); + } + + var resolvedPath = gpii.glob.sanitisePath(fluid.module.resolvePath(rootPath)); + var pathedIncludes = gpii.glob.addPathToPatterns(resolvedPath, includes); + var pathedExcludes = gpii.glob.addPathToPatterns(resolvedPath, excludes); + return gpii.glob.scanSingleDir(resolvedPath, pathedIncludes, pathedExcludes, minimatchOptions); +}; + +/** + * Scan a single directory level and return a list of files and sub-directories that match the includes and do not + * match the excludes. Each entry returned must: + * + * 1. Match at least one include. + * 2. Not match a "negated include". + * 3. Not match an exclude (or be allowed by a "negated exclude"). + * + * Each file encountered is added to the overall list if all of the above are true. Directories are handled a bit + * differently, as we attempt to interpret whether the directory MIGHT contain content that matches an include. If so, + * the directory is scanned using this same function, and any sub-matches are added to our results. + * + * @param {String} dirPath - A full path to the directory to scan. + * @param {Array} includes - An array of full or package-relative paths to include in the search results. + * @param {Array} excludes - An array of full or package-relative paths to exclude from the search results. + * @param {Object} [minimatchOptions] - (Optional) options to pass to minimatch. + * @return {Array} An array of matching paths. + * + */ +gpii.glob.scanSingleDir = function (dirPath, includes, excludes, minimatchOptions) { + var matchingPaths = []; + + var dirPaths = fs.readdirSync(dirPath).map(function (subPath) { return path.posix.resolve(dirPath, subPath); }).sort(); + + // Check to see if this path should be included or excluded + var allowedPaths = gpii.glob.filterPaths(dirPaths, includes, excludes, minimatchOptions); + + fluid.each(allowedPaths, function (singlePath) { + var itemStats = fs.statSync(singlePath); + if (itemStats.isDirectory()) { + var subMatches = gpii.glob.scanSingleDir(singlePath, includes, excludes, minimatchOptions); + if (subMatches.length) { + matchingPaths = matchingPaths.concat(subMatches); + } + } + else if (itemStats.isFile()) { + matchingPaths.push(singlePath); + } + }); + + return matchingPaths; +}; + +/** + * + * Filter a list of paths using "includes" and "excludes" and return paths that: + * + * 1. Match at least one (non-negated) include. + * 2. Do not match any negated includes. + * 3. Either: + * a. Do not match any (non-negated) excludes. + * b. Match a negated exclude. + * + * @param {Array} dirPaths - An array of full paths to check. + * @param {Array} includes - An array of full or package-relative paths to include in the search results. + * @param {Array} excludes - An array of full or package-relative paths to exclude from the search results. + * @param {Object} [minimatchOptions] - (Optional) options to pass to minimatch. + * @return {Array} An array of paths allowed by the include and exclude filters. + * + */ +gpii.glob.filterPaths = function (dirPaths, includes, excludes, minimatchOptions) { + var matchingPaths = []; + var positiveIncludes = gpii.glob.positivePatterns(includes); + var negativeIncludes = gpii.glob.negativePatterns(includes); + var positiveExcludes = gpii.glob.positivePatterns(excludes); + var negativeExcludes = gpii.glob.negativePatterns(excludes); + + fluid.each(dirPaths, function (singlePath) { + var stats = fs.statSync(singlePath); + var isDir = stats.isDirectory(); + + var matchesPositiveInclude = fluid.find(positiveIncludes, function (positivePattern) { + return gpii.glob.matchesSinglePattern(singlePath, positivePattern, minimatchOptions, isDir) || undefined; + }); + + if (matchesPositiveInclude) { + // Check negated excludes for a match. + var matchesNegatedExclude = fluid.find(negativeExcludes, function (negatedExcludePattern) { + return gpii.glob.matchesSinglePattern(singlePath, negatedExcludePattern, minimatchOptions, isDir) || undefined; + }); + + // Negated excludes trump excludes and negated includes. + if (matchesNegatedExclude) { + matchingPaths.push(singlePath); + } + // Check negated includes and regular excludes together. + else { + var combinedExcludes = negativeIncludes.concat(positiveExcludes); + var matchesExclude = fluid.find(combinedExcludes, function (excludePattern) { + // Excludes should not use the special handling for directories. + return gpii.glob.matchesSinglePattern(singlePath, excludePattern, minimatchOptions) || undefined; + }); + + if (!matchesExclude) { + matchingPaths.push(singlePath); + } + } + } + }); + + return matchingPaths; +}; + +/** + * + * Check a single path against a single "glob" pattern. + * + * @param {String} pathToMatch - A full path to evaluate. + * @param {String} pattern - A single "glob" pattern. + * @param {Object} [minimatchOptions] - (Optional) options to pass to minimatch. + * @param {Boolean} [isDir] - (Optional) Whether or not the path refers to a directory. + * @return {Boolean} `true` if the pattern matches, `false` if not. + * + */ +gpii.glob.matchesSinglePattern = function (pathToMatch, pattern, minimatchOptions, isDir) { + minimatchOptions = minimatchOptions || {}; + + if (isDir) { + return gpii.glob.dirMightMatch(pathToMatch, pattern); + } + else { + return minimatch(pathToMatch, pattern, minimatchOptions); + } +}; + +/** + * + * Match a directory against a pattern and return true if it might contain material that matches the pattern. + * + * @param {String} pathToDir - The full path to the directory. + * @param {String} pattern - The (positive) pattern to test the path against. + * @return {Boolean} `true` if the directory might contain matches, `false` otherwise. + * + */ +gpii.glob.dirMightMatch = function (pathToDir, pattern) { + // We use the equivalent of the basePath option in minimatch, i.e. any directory might contain a pattern with no slashes. + if (pattern.indexOf("/") !== -1) { + var patternSegments = pattern.split("/"); + var pathSegments = pathToDir.split(path.posix.sep); + + for (var a = 0; a < pathSegments.length; a++) { + var patternSegment = patternSegments[a]; + + // If we make it to a directory wildcard, there may be matches in the dir or one of its children. + if (patternSegment === "**") { + return true; + } + + var pathSegment = pathSegments[a]; + if (pathSegment !== patternSegment) { + return false; + } + } + } + + return true; +}; + +// The default list of regular expressions that describe "invalid globs". +gpii.glob.invalidGlobRules = { + noLeadingWildcard: { + message: "contains a leading wildcard", + pattern: /^(\.\/)?\*\*/ + }, + noWindowsSeparator: { + message: "contains a windows separator", + pattern: /\\/ + }, + noParentDir: { + message: "contains a reference to a parent directory", + pattern: /^\.\./ + }, + noRegexp: { + message: "contains a character used to define a regular expression", + pattern: /[\[\](){}|]/ + }, + noWholeRoot: { + message: "contains a reference to the whole of the root directory", + pattern: /^\.\/$/ + } +}; + +/** + * + * Check a pattern to ensure that it conforms to our constraints, which are: + * + * 1. It must not contain a leading wildcard, as in "**" or "./**". + * 2. It must not contain windows-style separators, i.e. backslashes. + * 3. It must not begin with a "parent" operator, i.e. "../" + * + * @param {String} pattern - A pattern to evaluate. + * @param {Object|Array} [rules] - An optional set of custom rules defining invalid patterns as regular expressions. + * @return {Array} An array of invalid patterns and details about why they are invalid.. + * + */ +gpii.glob.validatePattern = function (pattern, rules) { + var positivePattern = gpii.glob.positivePattern(pattern); + rules = rules || gpii.glob.invalidGlobRules; + + var failures = []; + fluid.each(rules, function (invalidGlobRule) { + if (positivePattern.match(invalidGlobRule.pattern)) { + failures.push({ + glob: positivePattern, + error: invalidGlobRule.message + }); + } + }); + + return failures; +}; + +/** + * + * Scan an entire array of patterns using gpii.glob.validatePattern (see above) and combine the results. + * + * @param {Array} patternArray - An array of patterns to evaluate. + * @param {Object|Array} [rules] - An optional set of custom rules defining invalid patterns as regular expressions. + * @return {Array} An array of invalid patterns and details about why they are invalid.. + * + */ +gpii.glob.validatePatternArray = function (patternArray, rules) { + var failures = []; + fluid.each(patternArray, function (pattern) { + failures = failures.concat(gpii.glob.validatePattern(pattern, rules)); + }); + return failures; +}; + +/** + * + * Log any invalid rules. + * + * @param {Array} violations - An array of violation objects, which contain a `glob` element (the failing pattern) and an `error` element (detailing why the pattern is invalid). + */ +gpii.glob.logInvalidRuleFeedback = function (violations) { + fluid.each(violations, function (violation) { + fluid.log("ERROR: Pattern '" + violation.glob + "' " + violation.error + "."); + }); +}; + +/** + * + * Create a callback function to filter an array for valid/invalid patterns using `gpii.glob.isValidPattern`. + * + * @param {Object|Array} [rules] - An optional set of custom rules defining invalid patterns as regular expressions. + * @param {Boolean} [showInvalid] - Set to true to include only invalid patterns. By default, valid patterns are returned. + * @return {Function} A callback function that can be used with `Array.filter()`. + * + */ +gpii.glob.makePatternFilter = function (rules, showInvalid) { + return function (pattern) { + var isValid = gpii.glob.isValidPattern(pattern, rules); + return showInvalid ? !isValid : isValid; + }; +}; + +/** + * + * Extract the "negative" patterns from an array of patterns, with the leading exclamation points removed. + * + * @param {Array} patterns - An array of patterns. + * @return {Array} All "negative" patterns, with their leading exclamation points removed. + * + */ +gpii.glob.negativePatterns = function (patterns) { + return patterns.filter(function (pattern) { + return pattern.indexOf("!") === 0; + }).map(function (pattern) { + return pattern.substring(1); + }); +}; + +/** + * + * Extract the "positive" patterns from an array of patterns. + * + * @param {Array} patterns - An array of patterns. + * @return {Array} Only the "positive" patterns. + * + */ +gpii.glob.positivePatterns = function (patterns) { + return patterns.filter(function (pattern) { + return pattern.indexOf("!") !== 0; + }); +}; + +/** + * + * Return the positive version of a pattern regardless of whether it is already positive or negative. + * + * @param {String} pattern - A glob pattern. + * @return {String} The pattern without any leading negation (!) operator. + * + */ +gpii.glob.positivePattern = function (pattern) { + return pattern.indexOf("!") === 0 ? pattern.substring(1) : pattern; +}; + +/** + * + * Add rootPath to relative paths, which are: + * + * 1. Paths that start with "./subdir/filename.js". + * 2. Paths that represent more than one directory level, as "subdir/filename.js" + * + * Note that as with minimatch itself, single-level patterns such as "filename.js" are left alone, so that they + * can be used to represent any file with a given name. + * + * @param {String} rootPath - The full path to the root. + * @param {Array} patterns - One or more patterns to prepend the path to. + * @return {Array} A copy of the original patterns with the path prepended to each. + * + */ +gpii.glob.addPathToPatterns = function (rootPath, patterns) { + // Ensure that the root path does not contain backslashes or a drive letter. + var sanitisedPath = gpii.glob.sanitisePath(rootPath); + + var pathedPatterns = fluid.transform(patterns, function (pattern) { + var positivePattern = gpii.glob.positivePattern(pattern); + var isNegated = positivePattern !== pattern; + var patternSegments = positivePattern.split("/"); + if (patternSegments.length > 1) { + var isFullPath = (patternSegments[0] === ""); + var firstSegment = isFullPath ? ("/" + patternSegments[1]) : patternSegments[0]; + var remainingSegments = patternSegments.slice(isFullPath ? 2 : 1); + // We explicitly resolve paths using the "posix" implementation on all platforms. + var resolvedPath = path.posix.resolve(sanitisedPath, firstSegment, remainingSegments.join("/")); + return (isNegated ? "!" : "") + resolvedPath; + } + // handle patterns like "file.js" + else { + return pattern; + } + }); + return pathedPatterns; +}; + +/** + * + * Convert windows-style paths (e.g. `c:\\path\\to\\filename.js`) to glob-compatible patterns, + * (e.g. `/path/to/filename.js`). + * + * @param {String} rawPath - The original path. + * @return {String} The sanitised path. + * + */ +gpii.glob.sanitisePath = function (rawPath) { + // Windows + if (rawPath.match(/[\:\\]/)) { + var pathSegments = rawPath.split(/[\/\\]+/); + var firstSegment = pathSegments[0].match(/^[a-zA-Z]:$/) ? "" : pathSegments[0]; + var sanitisedSegments = [firstSegment].concat(pathSegments.slice(1)); + return sanitisedSegments.join("/"); + } + // Everything else. + else { + return rawPath; + } +}; diff --git a/tests/all-tests.js b/tests/all-tests.js new file mode 100644 index 0000000..9eda1a0 --- /dev/null +++ b/tests/all-tests.js @@ -0,0 +1,4 @@ +"use strict"; + +require("./js/unit-tests"); +require("./js/functional-tests"); diff --git a/tests/find-fixture/.dot-file.js b/tests/find-fixture/.dot-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/.dot-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/node_modules/deep/deep-file.js b/tests/find-fixture/node_modules/deep/deep-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/node_modules/deep/deep-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/node_modules/deep/deeper/deeper-file.js b/tests/find-fixture/node_modules/deep/deeper/deeper-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/node_modules/deep/deeper/deeper-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/node_modules/root.js b/tests/find-fixture/node_modules/root.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/node_modules/root.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/root-file.js b/tests/find-fixture/root-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/root-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/src/deep/deep-file.js b/tests/find-fixture/src/deep/deep-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/src/deep/deep-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/src/deep/deeper/deeper-file.js b/tests/find-fixture/src/deep/deeper/deeper-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/src/deep/deeper/deeper-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/src/src-file.js b/tests/find-fixture/src/src-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/src/src-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/tests/deep/deep-file.js b/tests/find-fixture/tests/deep/deep-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/tests/deep/deep-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/tests/deep/deeper/deeper-file.js b/tests/find-fixture/tests/deep/deeper/deeper-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/tests/deep/deeper/deeper-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/find-fixture/tests/test-file.js b/tests/find-fixture/tests/test-file.js new file mode 100644 index 0000000..3918c74 --- /dev/null +++ b/tests/find-fixture/tests/test-file.js @@ -0,0 +1 @@ +"use strict"; diff --git a/tests/js/functional-tests.js b/tests/js/functional-tests.js new file mode 100644 index 0000000..8b492ab --- /dev/null +++ b/tests/js/functional-tests.js @@ -0,0 +1,175 @@ +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); +var jqUnit = require("node-jqunit"); +var path = require("path"); + +require("../../"); + +jqUnit.module("End-to-end tests for directory searching."); + +jqUnit.test("Test `findFiles` function.", function () { + var rootPath = fluid.module.resolvePath("%gpii-glob/tests/find-fixture"); + var testDefs = { + // TODO: Get these working, it fails but complains about the wrong number of expected assertions. + invalidInclude: { + message: "We should fail on an invalid include.", + includes: ["./**"], + excludes: [], + minimatchOptions: {}, + expectedErrors: ["One or more glob patterns you have entered are invalid. Cannot continue."] + }, + invalidExclude: { + message: "We should fail on an invalid exclude.", + includes: [], + excludes: ["c:\\"], + minimatchOptions: {}, + expectedErrors: ["One or more glob patterns you have entered are invalid. Cannot continue."] + }, + bothInvalid: { + message: "We should fail if there is both an invalid exclude and an invalid include.", + includes: ["**"], + excludes: ["./**"], + minimatchOptions: {}, + expectedErrors: ["One or more glob patterns you have entered are invalid. Cannot continue."] + }, + singleDirectoryWildcard: { + message: "We should be able to work with a directory wildcard.", + includes: ["./src/**/*.js"], + excludes: [], + minimatchOptions: {}, + expected: ["./src/deep/deep-file.js", "./src/deep/deeper/deeper-file.js", "./src/src-file.js"], + expectedErrors: [] + }, + nestedDirectoryWildcard: { + message: "We should be able to work with nested directory wildcards.", + includes: ["./src/**/deeper/*.js"], + excludes: [], + minimatchOptions: {}, + expected: ["./src/deep/deeper/deeper-file.js"], + expectedErrors: [] + }, + excludes: { + message: "We should be able to work with excludes.", + includes: ["./src/**/*.js"], + excludes: ["./src/**/deeper/*.js"], + minimatchOptions: {}, + expected: ["./src/deep/deep-file.js", "./src/src-file.js"], + expectedErrors: [] + }, + // TODO: There appears to be a bug in our logic that results in only using the first include. FIX. + multipleIncludes: { + message: "We should be able to work with multiple includes.", + includes: ["./src/*.js", "./src/**/deeper/*.js"], + excludes: [], + minimatchOptions: {}, + expected: ["./src/deep/deeper/deeper-file.js", "./src/src-file.js"], + expectedErrors: [] + }, + negatedInclude: { + message: "We should be able to work with negated includes.", + includes: ["./src/**/*.js", "!./src/**/deeper/*.js"], + excludes: [], + minimatchOptions: {}, + expected: ["./src/deep/deep-file.js", "./src/src-file.js"], + expectedErrors: [] + }, + negatedExclude: { + message: "We should be able to work with negated excludes.", + includes: ["./src/**/*.js"], + excludes: ["./src/deep/**/*.js", "!./src/**/deeper/*.js"], + minimatchOptions: {}, + expected: ["./src/deep/deeper/deeper-file.js", "./src/src-file.js"], + expectedErrors: [] + }, + onlyNegatedExclude: { + message: "Negated excludes should only affect included material.", + includes: [], + excludes: ["!./src/**/deeper/*.js"], + minimatchOptions: {}, + expected: [], + expectedErrors: [] + }, + matchBaseOption: { + message: "We should be able to work with a `matchBase`-style pattern.", + includes: ["deep-file.js"], + excludes: ["./node_modules/**/*.js"], + minimatchOptions: { matchBase: true }, + expected: ["./src/deep/deep-file.js", "./tests/deep/deep-file.js"], + expectedErrors: [] + }, + dotOption: { + message: "We should be able to include dotfiles in filename wildcard matching.", + includes: ["./*.js"], + excludes: [], + minimatchOptions: { dot: true }, + expected: ["./.dot-file.js", "./root-file.js"], + expectedErrors: [] + }, + noMinimatchOptions: { + message: "We should be able to work without custom minimatch options.", + includes: ["./*.js"], + excludes: [], + minimatchOptions: undefined, + expected: ["./root-file.js"], + expectedErrors: [] + }, + addCustomRules: { + message: "We should be able to add a custom 'invalid pattern' rule.", + includes: ["./package.json"], + excludes: [], + rules: { + rejectAll: { + message: "contains one or more characters", + pattern: /.+/ + } + }, + expectedErrors: ["One or more glob patterns you have entered are invalid. Cannot continue."] + }, + removeDefaultRules: { + message: "We should be able to remove a default 'invalid pattern' rule.", + includes: ["./**/deep-file.js"], + excludes: [], + rules: {}, + expected: [ + "./node_modules/deep/deep-file.js", + "./src/deep/deep-file.js", + "./tests/deep/deep-file.js" + ] + } + }; + + /* + We use expectFrameworkDiagnostic for our "failure" tests, but not for others. That function + calls expect and in essence increments the expected failure count on its own. + + https://github.com/fluid-project/infusion/blob/master/tests/test-core/jqUnit/js/jqUnit.js#L279 + + That count starts at 0 by default. So, we have to tell it about the non-failures, but let it handle + incrementing "expect" for the failures. + + TODO: Discuss this with Antranig + */ + jqUnit.expect(Object.keys(testDefs).length - 4); // exclude the four tests that should fail from the expect count. + fluid.each(testDefs, function (testDef) { + if (testDef.expectedErrors && testDef.expectedErrors.length) { + jqUnit.expectFrameworkDiagnostic( + testDef.message, + function () { + gpii.glob.findFiles(rootPath, testDef.includes, testDef.excludes, testDef.minimatchOptions, testDef.rules); + }, + testDef.expectedErrors + ); + } + else { + var output = gpii.glob.findFiles(rootPath, testDef.includes, testDef.excludes, testDef.minimatchOptions, testDef.rules); + + // The output will always be full paths, so we need to add the root path to our expected output. + var pathedExpected = testDef.expected.map(function (singlePath) { + return path.posix.resolve(gpii.glob.sanitisePath(rootPath), singlePath); + }); + + jqUnit.assertDeepEq(testDef.message, pathedExpected, output); + } + }); +}); diff --git a/tests/js/unit-tests.js b/tests/js/unit-tests.js new file mode 100644 index 0000000..7589b40 --- /dev/null +++ b/tests/js/unit-tests.js @@ -0,0 +1,229 @@ +"use strict"; +var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); +var jqUnit = require("node-jqunit"); + +require("../../"); + +jqUnit.module("Unit tests for gpii-glob package."); + +jqUnit.test("Test positive and negative pattern filtering.", function () { + var testDefs = { + empty: { + message: "An empty array should be preserved", + input: [], + expectedPositive: [], + expectedNegative: [] + }, + mixed: { + message: "An array of mixed entries should be filtered as expected", + input: ["!negative", "positive"], + expectedPositive: ["positive"], + expectedNegative: ["negative"] + } + }; + + fluid.each(testDefs, function (testDef) { + var positives = gpii.glob.positivePatterns(testDef.input); + jqUnit.assertDeepEq(testDef.message + ": positive matches", testDef.expectedPositive, positives); + + var negatives = gpii.glob.negativePatterns(testDef.input); + jqUnit.assertDeepEq(testDef.message + ": negative matches", testDef.expectedNegative, negatives); + }); +}); + +jqUnit.test("Test pattern validity checks.", function () { + var validPatterns = [ + "./src/**/*.js", + "!./src/**/*.js", + "./full/path/to/file.js", + "./relative/path/to/file.js", + "./src/../filename.js" + ]; + var invalidPatterns = [ + "./**", + "./**/*.js", + "**", + "**/*.js", + "!**/*.js", + "!./**/*.js", + "../filename.js", + "./(this|that)/**/*.js" + ]; + + // Scan valid patterns with the default rules. + fluid.each(validPatterns, function (validPattern) { + var violations = gpii.glob.validatePattern(validPattern); + jqUnit.assertTrue("A valid pattern should be valid.", violations.length === 0); + }); + + // Scan invalid patterns with the default rules. + fluid.each(invalidPatterns, function (invalidPattern) { + var violations = gpii.glob.validatePattern(invalidPattern); + jqUnit.assertTrue("An invalid pattern should not be valid.", violations.length > 0); + }); + + var strictRules = { + rejectAll: { + message: "contains one or more characters", + pattern: /.+/ + } + }; + + // Scan valid patterns with custom rules that make them invalid. + fluid.each(validPatterns, function (validPattern) { + var violations = gpii.glob.validatePattern(validPattern, strictRules); + jqUnit.assertTrue("We should be able to add a custom rule when checking validity.", violations.length > 0); + }); + + var noRules = {}; + + // Scan invalid patterns with custom rules that disable all checks. + fluid.each(invalidPatterns, function (invalidPattern) { + var violations = gpii.glob.validatePattern(invalidPattern, noRules); + jqUnit.assertTrue("We should be able to remove default rules when testing validity.", violations.length === 0); + }); +}); + +jqUnit.test("Test single pattern matching.", function () { + var testDefs = { + filenameWildcard: { + message: "We should be able to match based on filename wildcards.", + positive: ["filename.js", "other.js"], + negative: ["README.md", "UPPERCASE.JS"], + pattern: "*.js" + }, + leadingDot: { + message: "We should be able to match filenames with a leading dot.", + positive: [".gitignore", ".eslintrc.json"], + negative: ["README.md", "filename.js"], + pattern: ".*" + }, + dirWildCard: { + message: "We should be able to match based on directory wildcards.", + positive: ["./src/deep/path/filename.js", "./src/shallow.js"], + negative: ["./root.js", "./lib/filename.js", "./lib/deep/src/filename.js"], + pattern: "./src/**/*.js" + }, + multipleDirWildcards: { + message: "We should be able to match based on multiple directory wildcards.", + positive: ["./src/lib/filename.js", "./src/deep/lib/filename.js"], + negative: ["./src/filename.js", "./lib/src/filename.js", "./src/deep/filename.js"], + pattern: "./src/**/lib/*.js" + }, + fullPath: { + message: "We should be able to handle full paths.", + positive: ["/path/to/filename.js"], + negative: ["filename.js", "path/to/filename.js"], + pattern: "/path/to/filename.js" + }, + relativePath: { + message: "We should be able to handle relative paths.", + positive: ["relative/path/to/content.txt"], + negative: ["./relative/path/to/content.txt", "/relative/path/to/content.txt"], + pattern: "relative/path/to/content.txt" + } + }; + + fluid.each(testDefs, function (testDef) { + fluid.each(fluid.makeArray(testDef.positive), function (shouldMatch) { + jqUnit.assertTrue(testDef.message + ": positive matching", gpii.glob.matchesSinglePattern(shouldMatch, testDef.pattern)); + }); + + fluid.each(fluid.makeArray(testDef.negative), function (shouldNotMatch) { + jqUnit.assertFalse(testDef.message + ": negative matching", gpii.glob.matchesSinglePattern(shouldNotMatch, testDef.pattern)); + }); + }); +}); + +jqUnit.test("Test `sanitisePath` function", function () { + var testDefs = { + fullWindows: { + message: "We should be able to handle a full windows path, including backslashes.", + input: "c:\\path\\to\\filename.js", + expected: "/path/to/filename.js" + }, + halfWindows: { + message: "We should be able to handle a 'half' windows path, with a drive letter, but leading slashes.", + input: "c:/path/to/filename.js", + expected: "/path/to/filename.js" + }, + windowsRoot: { + message: "We should be able to handle a drive root", + input: "c:\\", + expected: "/" + }, + nonWindows: { + message: "We should be able to handle a non-windows path.", + input: "/path/to/filename.js", + expected: "/path/to/filename.js" + } + }; + + fluid.each(testDefs, function (testDef) { + var output = gpii.glob.sanitisePath(testDef.input); + jqUnit.assertEquals(testDef.message, testDef.expected, output); + }); +}); + +jqUnit.test("Test `addPathToPatterns` function.", function () { + var testDefs = { + withLeadingDot: { + message: "We should be able to handle relative paths with a leading dot.", + rootPath: "/root", + patterns: ["./path/to/filename.js", "./README.md", "!./.gitignore"], + expected: ["/root/path/to/filename.js", "/root/README.md", "!/root/.gitignore"] + }, + withoutLeadingDot: { + message: "We should be able to handle relative paths without a leading dot.", + rootPath: "/root", + patterns: ["path/to/filename.js", "!path/to/lib/filename.js"], + expected: ["/root/path/to/filename.js", "!/root/path/to/lib/filename.js"] + }, + fullPath: { + message: "We should be able to handle full paths.", + rootPath: "/root", + patterns: ["/otherRoot/path/to/filename.js", "!/otherRoot/exclude.js"], + expected: ["/otherRoot/path/to/filename.js", "!/otherRoot/exclude.js"] + }, + onlyFilename: { + message: "We should be able to handle a path that only consists of a filename.", + rootPath: "/root", + patterns: ["README.md", "!.gitignore"], + expected: ["README.md", "!.gitignore"] + } + }; + + fluid.each(testDefs, function (testDef) { + var output = gpii.glob.addPathToPatterns(testDef.rootPath, testDef.patterns); + jqUnit.assertDeepEq(testDef.message, testDef.expected, output); + }); +}); + +jqUnit.test("Test `dirMightMatch` function.", function () { + var testDefs = { + filenameWildcard: { + message: "We should be able to match a pattern that contains filename globbing.", + pattern: "/root/path/*.js", + hits: ["/root/path"], + misses: ["/root/other/path"] + }, + dirWildCard: { + message: "We should be able to match a pattern that contains directory globbing.", + pattern: "/root/src/**/*.js", + hits: ["/root/src", "/root/src/js"], + misses: ["/root/tests", "/root/node_modules/module/src"] + } + }; + + fluid.each(testDefs, function (testDef) { + fluid.each(testDef.hits, function (shouldMatch, index) { + var matches = gpii.glob.dirMightMatch(shouldMatch, testDef.pattern); + jqUnit.assertTrue(testDef.message + ": hit " + index, matches); + }); + fluid.each(testDef.misses, function (shouldNotMatch, index) { + var matches = gpii.glob.dirMightMatch(shouldNotMatch, testDef.pattern); + jqUnit.assertFalse(testDef.message + ": miss " + index, matches); + }); + }); +});