Merge pull request #386 from snyk-tech-services/feat/find-files

feat: find files on disk with filters & ignores

package.json

@@ -47,6 +47,7 @@
     "bunyan": "1.8.15",
     "debug": "4.3.4",
     "lodash": "4.17.21",
+    "micromatch": "4.0.5",
     "needle": "2.9.1",
     "p-map": "4.0.0",
     "parse-link-header": "2.0.0",
@@ -63,6 +64,7 @@
     "@types/debug": "4.1.5",
     "@types/jest": "^25.1.1",
     "@types/lodash": "^4.14.149",
+    "@types/micromatch": "4.0.2",
     "@types/needle": "2.0.4",
     "@types/node": "^12.12.26",
     "@types/parse-link-header": "1.0.0",

src/lib/find-files.ts

@@ -0,0 +1,158 @@
+import * as fs from 'fs';
+import * as micromatch from 'micromatch';
+import * as pathLib from 'path';
+import * as debugModule from 'debug';
+const debug = debugModule('snyk:find-files');
+
+/**
+ * Returns files inside given file path.
+ *
+ * @param path file path.
+ */
+export async function readDirectory(path: string): Promise<string[]> {
+  return await new Promise((resolve, reject) => {
+    fs.readdir(path, (err, files) => {
+      if (err) {
+        reject(err);
+      }
+      resolve(files);
+    });
+  });
+}
+
+/**
+ * Returns file stats object for given file path.
+ *
+ * @param path path to file or directory.
+ */
+export async function getStats(path: string): Promise<fs.Stats> {
+  return await new Promise((resolve, reject) => {
+    fs.stat(path, (err, stats) => {
+      if (err) {
+        reject(err);
+      }
+      resolve(stats);
+    });
+  });
+}
+
+interface FindFilesRes {
+  files: string[];
+  allFilesFound: string[];
+}
+
+/**
+ * Find all files in given search path. Returns paths to files found.
+ *
+ * @param path file path to search.
+ * @param ignore (optional) globs to ignore. Will always ignore node_modules.
+ * @param filter (optional) file names to find. If not provided all files are returned.
+ * @param levelsDeep (optional) how many levels deep to search, defaults to 5, this path and one sub directory.
+ */
+export async function find(
+  path: string,
+  ignore: string[] = [],
+  filter: string[] = [],
+  levelsDeep = 5,
+): Promise<FindFilesRes> {
+  const found: string[] = [];
+  const foundAll: string[] = [];
+
+  // ensure we ignore find against node_modules path.
+  if (path.endsWith('node_modules')) {
+    return { files: found, allFilesFound: foundAll };
+  }
+  // ensure node_modules is always ignored
+  if (!ignore.includes('node_modules')) {
+    ignore.push('node_modules');
+  }
+  try {
+    if (levelsDeep < 0) {
+      return { files: found, allFilesFound: foundAll };
+    } else {
+      levelsDeep--;
+    }
+    const fileStats = await getStats(path);
+    if (fileStats.isDirectory()) {
+      const { files, allFilesFound } = await findInDirectory(
+        path,
+        ignore,
+        filter,
+        levelsDeep,
+      );
+      found.push(...files);
+      foundAll.push(...allFilesFound);
+    } else if (fileStats.isFile()) {
+      const fileFound = findFile(path, filter, ignore);
+      if (fileFound) {
+        found.push(fileFound);
+        foundAll.push(fileFound);
+      }
+    }
+    const filteredOutFiles = foundAll.filter((f) => !found.includes(f));
+    if (filteredOutFiles.length) {
+      debug(
+        `Filtered out ${filteredOutFiles.length}/${
+          foundAll.length
+        } files: ${filteredOutFiles.join(', ')}`,
+      );
+    }
+    return { files: found, allFilesFound: foundAll };
+  } catch (err) {
+    throw new Error(`Error finding files in path '${path}'.\n${err.message}`);
+  }
+}
+
+function findFile(
+  path: string,
+  filter: string[] = [],
+  ignore: string[] = [],
+): string | null {
+  if (filter.length > 0) {
+    const filename = pathLib.basename(path);
+    if (matches(filename, filter)) {
+      return path;
+    }
+  } else {
+    if (matches(path, ignore)) {
+      return null;
+    }
+    return path;
+  }
+  return null;
+}
+
+async function findInDirectory(
+  path: string,
+  ignore: string[] = [],
+  filter: string[] = [],
+  levelsDeep = 4,
+): Promise<FindFilesRes> {
+  const files = await readDirectory(path);
+  const toFind = files
+    .filter((file) => !matches(file, ignore))
+    .map((file) => {
+      const resolvedPath = pathLib.resolve(path, file);
+      if (!fs.existsSync(resolvedPath)) {
+        debug('File does not seem to exist, skipping: ', file);
+        return { files: [], allFilesFound: [] };
+      }
+      return find(resolvedPath, ignore, filter, levelsDeep);
+    });
+
+  const found = await Promise.all(toFind);
+  return {
+    files: Array.prototype.concat.apply(
+      [],
+      found.map((f) => f.files),
+    ),
+    allFilesFound: Array.prototype.concat.apply(
+      [],
+      found.map((f) => f.allFilesFound),
+    ),
+  };
+}
+
+function matches(filePath: string, globs: string[]): boolean {
+  return globs.some((glob) => micromatch.isMatch(filePath, glob));
+}

src/lib/index.ts

@@ -11,6 +11,7 @@ export * from './get-logging-path';
 export * from './get-snyk-host';
 export * from './filter-out-existing-orgs';
 export * from './supported-project-types';
+export * from './find-files';
 
 export * from './source-handlers/github';
 export * from './source-handlers/gitlab';

test/lib/find-files.test.ts

@@ -0,0 +1,83 @@
+import * as path from 'path';
+
+import { find } from '../../src/lib';
+
+export function getFixturePath(fixtureName: string): string {
+  return path.join(__dirname, './fixtures', fixtureName);
+}
+const testFixture = getFixturePath('find-files');
+
+test('find path is empty string', async () => {
+  await expect(find('')).rejects.toThrowError("Error finding files in path ''");
+});
+
+test('find path that does not exist', async () => {
+  await expect(find('does-not-exist')).rejects.toThrowError(
+    "Error finding files in path 'does-not-exist'",
+  );
+});
+
+test('find all files in test fixture ignoring node_modules', async () => {
+  // six levels deep to ensure node_modules is tested
+  const { files: result } = await find(testFixture, ['node_modules'], [], 6);
+  const expected = [
+    path.join(testFixture, 'maven', 'pom.xml'),
+    path.join(testFixture, 'maven', 'test.txt'),
+    path.join(testFixture, 'mvn', 'pom.xml'),
+    path.join(testFixture, 'mvn', 'test.txt'),
+  ];
+  expect(result.sort()).toStrictEqual(expected.sort());
+}, 5000);
+
+test('find all files in test fixture ignoring node_modules by default', async () => {
+  // six levels deep to ensure node_modules is tested
+  const { files: result } = await find(testFixture, [], [], 6);
+  const expected = [
+    path.join(testFixture, 'maven', 'pom.xml'),
+    path.join(testFixture, 'maven', 'test.txt'),
+    path.join(testFixture, 'mvn', 'pom.xml'),
+    path.join(testFixture, 'mvn', 'test.txt'),
+  ];
+  expect(result.sort()).toStrictEqual(expected.sort());
+}, 5000);
+
+test('find all files in test fixture ignoring *.txt', async () => {
+  // six levels deep to ensure node_modules is tested
+  const { files: result } = await find(testFixture, ['*.txt'], [], 6);
+  const expected = [
+    path.join(testFixture, 'maven', 'pom.xml'),
+    path.join(testFixture, 'mvn', 'pom.xml'),
+  ];
+  expect(result.sort()).toStrictEqual(expected.sort());
+}, 5000);
+
+test('find all files in test fixture by filtering for *.xml', async () => {
+  // six levels deep to ensure node_modules is tested
+  const { files: result } = await find(testFixture, [], ['*.xml'], 6);
+  const expected = [
+    path.join(testFixture, 'maven', 'pom.xml'),
+    path.join(testFixture, 'mvn', 'pom.xml'),
+  ];
+  expect(result.sort()).toStrictEqual(expected.sort());
+}, 5000);
+
+test('find all files in test fixture but filter out specific path for mvn', async () => {
+  const { files: result } = await find(testFixture, ['**/mvn/**.xml'], [], 6);
+  const expected = [
+    path.join(testFixture, 'maven', 'pom.xml'),
+    path.join(testFixture, 'maven', 'test.txt'),
+    path.join(testFixture, 'mvn', 'test.txt'),
+  ];
+  expect(result.sort()).toStrictEqual(expected.sort());
+}, 5000);
+
+test.todo('Test **/folder/*.txt');
+
+test('find pom.xml files in test fixture', async () => {
+  const { files: result } = await find(testFixture, [], ['pom.xml']);
+  const expected = [
+    path.join(testFixture, 'maven', 'pom.xml'),
+    path.join(testFixture, 'mvn', 'pom.xml'),
+  ];
+  expect(result.sort()).toStrictEqual(expected);
+});