From 476e8e4f5fb2030345c13e80f6fd20583881cc8f Mon Sep 17 00:00:00 2001 From: ghe Date: Thu, 17 Nov 2022 22:29:55 +0000 Subject: [PATCH] feat: find files on disk with filters & ignores --- package.json | 2 + src/lib/find-files.ts | 158 ++++++++++++++++++++ src/lib/index.ts | 1 + test/lib/find-files.test.ts | 83 ++++++++++ test/lib/fixtures/find-files/maven/pom.xml | 0 test/lib/fixtures/find-files/maven/test.txt | 0 test/lib/fixtures/find-files/mvn/pom.xml | 0 test/lib/fixtures/find-files/mvn/test.txt | 0 8 files changed, 244 insertions(+) create mode 100644 src/lib/find-files.ts create mode 100644 test/lib/find-files.test.ts create mode 100644 test/lib/fixtures/find-files/maven/pom.xml create mode 100644 test/lib/fixtures/find-files/maven/test.txt create mode 100644 test/lib/fixtures/find-files/mvn/pom.xml create mode 100644 test/lib/fixtures/find-files/mvn/test.txt diff --git a/package.json b/package.json index 1243b057..70fc3da2 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "bunyan": "1.8.15", "debug": "4.3.4", "lodash": "4.17.21", + "micromatch": "4.0.5", "needle": "2.9.1", "p-map": "4.0.0", "parse-link-header": "2.0.0", @@ -63,6 +64,7 @@ "@types/debug": "4.1.5", "@types/jest": "^25.1.1", "@types/lodash": "^4.14.149", + "@types/micromatch": "4.0.2", "@types/needle": "2.0.4", "@types/node": "^12.12.26", "@types/parse-link-header": "1.0.0", diff --git a/src/lib/find-files.ts b/src/lib/find-files.ts new file mode 100644 index 00000000..591629b9 --- /dev/null +++ b/src/lib/find-files.ts @@ -0,0 +1,158 @@ +import * as fs from 'fs'; +import * as micromatch from 'micromatch'; +import * as pathLib from 'path'; +import * as debugModule from 'debug'; +const debug = debugModule('snyk:find-files'); + +/** + * Returns files inside given file path. + * + * @param path file path. + */ +export async function readDirectory(path: string): Promise { + return await new Promise((resolve, reject) => { + fs.readdir(path, (err, files) => { + if (err) { + reject(err); + } + resolve(files); + }); + }); +} + +/** + * Returns file stats object for given file path. + * + * @param path path to file or directory. + */ +export async function getStats(path: string): Promise { + return await new Promise((resolve, reject) => { + fs.stat(path, (err, stats) => { + if (err) { + reject(err); + } + resolve(stats); + }); + }); +} + +interface FindFilesRes { + files: string[]; + allFilesFound: string[]; +} + +/** + * Find all files in given search path. Returns paths to files found. + * + * @param path file path to search. + * @param ignore (optional) globs to ignore. Will always ignore node_modules. + * @param filter (optional) file names to find. If not provided all files are returned. + * @param levelsDeep (optional) how many levels deep to search, defaults to 5, this path and one sub directory. + */ +export async function find( + path: string, + ignore: string[] = [], + filter: string[] = [], + levelsDeep = 5, +): Promise { + const found: string[] = []; + const foundAll: string[] = []; + + // ensure we ignore find against node_modules path. + if (path.endsWith('node_modules')) { + return { files: found, allFilesFound: foundAll }; + } + // ensure node_modules is always ignored + if (!ignore.includes('node_modules')) { + ignore.push('node_modules'); + } + try { + if (levelsDeep < 0) { + return { files: found, allFilesFound: foundAll }; + } else { + levelsDeep--; + } + const fileStats = await getStats(path); + if (fileStats.isDirectory()) { + const { files, allFilesFound } = await findInDirectory( + path, + ignore, + filter, + levelsDeep, + ); + found.push(...files); + foundAll.push(...allFilesFound); + } else if (fileStats.isFile()) { + const fileFound = findFile(path, filter, ignore); + if (fileFound) { + found.push(fileFound); + foundAll.push(fileFound); + } + } + const filteredOutFiles = foundAll.filter((f) => !found.includes(f)); + if (filteredOutFiles.length) { + debug( + `Filtered out ${filteredOutFiles.length}/${ + foundAll.length + } files: ${filteredOutFiles.join(', ')}`, + ); + } + return { files: found, allFilesFound: foundAll }; + } catch (err) { + throw new Error(`Error finding files in path '${path}'.\n${err.message}`); + } +} + +function findFile( + path: string, + filter: string[] = [], + ignore: string[] = [], +): string | null { + if (filter.length > 0) { + const filename = pathLib.basename(path); + if (matches(filename, filter)) { + return path; + } + } else { + if (matches(path, ignore)) { + return null; + } + return path; + } + return null; +} + +async function findInDirectory( + path: string, + ignore: string[] = [], + filter: string[] = [], + levelsDeep = 4, +): Promise { + const files = await readDirectory(path); + const toFind = files + .filter((file) => !matches(file, ignore)) + .map((file) => { + const resolvedPath = pathLib.resolve(path, file); + if (!fs.existsSync(resolvedPath)) { + debug('File does not seem to exist, skipping: ', file); + return { files: [], allFilesFound: [] }; + } + return find(resolvedPath, ignore, filter, levelsDeep); + }); + + const found = await Promise.all(toFind); + return { + files: Array.prototype.concat.apply( + [], + found.map((f) => f.files), + ), + allFilesFound: Array.prototype.concat.apply( + [], + found.map((f) => f.allFilesFound), + ), + }; +} + +function matches(filePath: string, globs: string[]): boolean { + return globs.some((glob) => micromatch.isMatch(filePath, glob)); +} diff --git a/src/lib/index.ts b/src/lib/index.ts index c80b2b04..d9a04423 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -11,6 +11,7 @@ export * from './get-logging-path'; export * from './get-snyk-host'; export * from './filter-out-existing-orgs'; export * from './supported-project-types'; +export * from './find-files'; export * from './source-handlers/github'; export * from './source-handlers/gitlab'; diff --git a/test/lib/find-files.test.ts b/test/lib/find-files.test.ts new file mode 100644 index 00000000..91f180af --- /dev/null +++ b/test/lib/find-files.test.ts @@ -0,0 +1,83 @@ +import * as path from 'path'; + +import { find } from '../../src/lib'; + +export function getFixturePath(fixtureName: string): string { + return path.join(__dirname, './fixtures', fixtureName); +} +const testFixture = getFixturePath('find-files'); + +test('find path is empty string', async () => { + await expect(find('')).rejects.toThrowError("Error finding files in path ''"); +}); + +test('find path that does not exist', async () => { + await expect(find('does-not-exist')).rejects.toThrowError( + "Error finding files in path 'does-not-exist'", + ); +}); + +test('find all files in test fixture ignoring node_modules', async () => { + // six levels deep to ensure node_modules is tested + const { files: result } = await find(testFixture, ['node_modules'], [], 6); + const expected = [ + path.join(testFixture, 'maven', 'pom.xml'), + path.join(testFixture, 'maven', 'test.txt'), + path.join(testFixture, 'mvn', 'pom.xml'), + path.join(testFixture, 'mvn', 'test.txt'), + ]; + expect(result.sort()).toStrictEqual(expected.sort()); +}, 5000); + +test('find all files in test fixture ignoring node_modules by default', async () => { + // six levels deep to ensure node_modules is tested + const { files: result } = await find(testFixture, [], [], 6); + const expected = [ + path.join(testFixture, 'maven', 'pom.xml'), + path.join(testFixture, 'maven', 'test.txt'), + path.join(testFixture, 'mvn', 'pom.xml'), + path.join(testFixture, 'mvn', 'test.txt'), + ]; + expect(result.sort()).toStrictEqual(expected.sort()); +}, 5000); + +test('find all files in test fixture ignoring *.txt', async () => { + // six levels deep to ensure node_modules is tested + const { files: result } = await find(testFixture, ['*.txt'], [], 6); + const expected = [ + path.join(testFixture, 'maven', 'pom.xml'), + path.join(testFixture, 'mvn', 'pom.xml'), + ]; + expect(result.sort()).toStrictEqual(expected.sort()); +}, 5000); + +test('find all files in test fixture by filtering for *.xml', async () => { + // six levels deep to ensure node_modules is tested + const { files: result } = await find(testFixture, [], ['*.xml'], 6); + const expected = [ + path.join(testFixture, 'maven', 'pom.xml'), + path.join(testFixture, 'mvn', 'pom.xml'), + ]; + expect(result.sort()).toStrictEqual(expected.sort()); +}, 5000); + +test('find all files in test fixture but filter out specific path for mvn', async () => { + const { files: result } = await find(testFixture, ['**/mvn/**.xml'], [], 6); + const expected = [ + path.join(testFixture, 'maven', 'pom.xml'), + path.join(testFixture, 'maven', 'test.txt'), + path.join(testFixture, 'mvn', 'test.txt'), + ]; + expect(result.sort()).toStrictEqual(expected.sort()); +}, 5000); + +test.todo('Test **/folder/*.txt'); + +test('find pom.xml files in test fixture', async () => { + const { files: result } = await find(testFixture, [], ['pom.xml']); + const expected = [ + path.join(testFixture, 'maven', 'pom.xml'), + path.join(testFixture, 'mvn', 'pom.xml'), + ]; + expect(result.sort()).toStrictEqual(expected); +}); diff --git a/test/lib/fixtures/find-files/maven/pom.xml b/test/lib/fixtures/find-files/maven/pom.xml new file mode 100644 index 00000000..e69de29b diff --git a/test/lib/fixtures/find-files/maven/test.txt b/test/lib/fixtures/find-files/maven/test.txt new file mode 100644 index 00000000..e69de29b diff --git a/test/lib/fixtures/find-files/mvn/pom.xml b/test/lib/fixtures/find-files/mvn/pom.xml new file mode 100644 index 00000000..e69de29b diff --git a/test/lib/fixtures/find-files/mvn/test.txt b/test/lib/fixtures/find-files/mvn/test.txt new file mode 100644 index 00000000..e69de29b