Skip to content

Commit

Permalink
feat: find files on disk with filters & ignores
Browse files Browse the repository at this point in the history
  • Loading branch information
lili2311 committed Nov 17, 2022
1 parent ec0ca43 commit 476e8e4
Show file tree
Hide file tree
Showing 8 changed files with 244 additions and 0 deletions.
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
"bunyan": "1.8.15",
"debug": "4.3.4",
"lodash": "4.17.21",
"micromatch": "4.0.5",
"needle": "2.9.1",
"p-map": "4.0.0",
"parse-link-header": "2.0.0",
Expand All @@ -63,6 +64,7 @@
"@types/debug": "4.1.5",
"@types/jest": "^25.1.1",
"@types/lodash": "^4.14.149",
"@types/micromatch": "4.0.2",
"@types/needle": "2.0.4",
"@types/node": "^12.12.26",
"@types/parse-link-header": "1.0.0",
Expand Down
158 changes: 158 additions & 0 deletions src/lib/find-files.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
import * as fs from 'fs';
import * as micromatch from 'micromatch';
import * as pathLib from 'path';
import * as debugModule from 'debug';
const debug = debugModule('snyk:find-files');

/**
* Returns files inside given file path.
*
* @param path file path.
*/
export async function readDirectory(path: string): Promise<string[]> {
return await new Promise((resolve, reject) => {
fs.readdir(path, (err, files) => {
if (err) {
reject(err);
}
resolve(files);
});
});
}

/**
* Returns file stats object for given file path.
*
* @param path path to file or directory.
*/
export async function getStats(path: string): Promise<fs.Stats> {
return await new Promise((resolve, reject) => {
fs.stat(path, (err, stats) => {
if (err) {
reject(err);
}
resolve(stats);
});
});
}

interface FindFilesRes {
files: string[];
allFilesFound: string[];
}

/**
* Find all files in given search path. Returns paths to files found.
*
* @param path file path to search.
* @param ignore (optional) globs to ignore. Will always ignore node_modules.
* @param filter (optional) file names to find. If not provided all files are returned.
* @param levelsDeep (optional) how many levels deep to search, defaults to 5, this path and one sub directory.
*/
export async function find(
path: string,
ignore: string[] = [],
filter: string[] = [],
levelsDeep = 5,
): Promise<FindFilesRes> {
const found: string[] = [];
const foundAll: string[] = [];

// ensure we ignore find against node_modules path.
if (path.endsWith('node_modules')) {
return { files: found, allFilesFound: foundAll };
}
// ensure node_modules is always ignored
if (!ignore.includes('node_modules')) {
ignore.push('node_modules');
}
try {
if (levelsDeep < 0) {
return { files: found, allFilesFound: foundAll };
} else {
levelsDeep--;
}
const fileStats = await getStats(path);
if (fileStats.isDirectory()) {
const { files, allFilesFound } = await findInDirectory(
path,
ignore,
filter,
levelsDeep,
);
found.push(...files);
foundAll.push(...allFilesFound);
} else if (fileStats.isFile()) {
const fileFound = findFile(path, filter, ignore);
if (fileFound) {
found.push(fileFound);
foundAll.push(fileFound);
}
}
const filteredOutFiles = foundAll.filter((f) => !found.includes(f));
if (filteredOutFiles.length) {
debug(
`Filtered out ${filteredOutFiles.length}/${
foundAll.length
} files: ${filteredOutFiles.join(', ')}`,
);
}
return { files: found, allFilesFound: foundAll };
} catch (err) {
throw new Error(`Error finding files in path '${path}'.\n${err.message}`);
}
}

function findFile(
path: string,
filter: string[] = [],
ignore: string[] = [],
): string | null {
if (filter.length > 0) {
const filename = pathLib.basename(path);
if (matches(filename, filter)) {
return path;
}
} else {
if (matches(path, ignore)) {
return null;
}
return path;
}
return null;
}

async function findInDirectory(
path: string,
ignore: string[] = [],
filter: string[] = [],
levelsDeep = 4,
): Promise<FindFilesRes> {
const files = await readDirectory(path);
const toFind = files
.filter((file) => !matches(file, ignore))
.map((file) => {
const resolvedPath = pathLib.resolve(path, file);
if (!fs.existsSync(resolvedPath)) {
debug('File does not seem to exist, skipping: ', file);
return { files: [], allFilesFound: [] };
}
return find(resolvedPath, ignore, filter, levelsDeep);
});

const found = await Promise.all(toFind);
return {
files: Array.prototype.concat.apply(
[],
found.map((f) => f.files),
),
allFilesFound: Array.prototype.concat.apply(
[],
found.map((f) => f.allFilesFound),
),
};
}

function matches(filePath: string, globs: string[]): boolean {
return globs.some((glob) => micromatch.isMatch(filePath, glob));
}
1 change: 1 addition & 0 deletions src/lib/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export * from './get-logging-path';
export * from './get-snyk-host';
export * from './filter-out-existing-orgs';
export * from './supported-project-types';
export * from './find-files';

export * from './source-handlers/github';
export * from './source-handlers/gitlab';
Expand Down
83 changes: 83 additions & 0 deletions test/lib/find-files.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import * as path from 'path';

import { find } from '../../src/lib';

export function getFixturePath(fixtureName: string): string {
return path.join(__dirname, './fixtures', fixtureName);
}
const testFixture = getFixturePath('find-files');

test('find path is empty string', async () => {
await expect(find('')).rejects.toThrowError("Error finding files in path ''");
});

test('find path that does not exist', async () => {
await expect(find('does-not-exist')).rejects.toThrowError(
"Error finding files in path 'does-not-exist'",
);
});

test('find all files in test fixture ignoring node_modules', async () => {
// six levels deep to ensure node_modules is tested
const { files: result } = await find(testFixture, ['node_modules'], [], 6);
const expected = [
path.join(testFixture, 'maven', 'pom.xml'),
path.join(testFixture, 'maven', 'test.txt'),
path.join(testFixture, 'mvn', 'pom.xml'),
path.join(testFixture, 'mvn', 'test.txt'),
];
expect(result.sort()).toStrictEqual(expected.sort());
}, 5000);

test('find all files in test fixture ignoring node_modules by default', async () => {
// six levels deep to ensure node_modules is tested
const { files: result } = await find(testFixture, [], [], 6);
const expected = [
path.join(testFixture, 'maven', 'pom.xml'),
path.join(testFixture, 'maven', 'test.txt'),
path.join(testFixture, 'mvn', 'pom.xml'),
path.join(testFixture, 'mvn', 'test.txt'),
];
expect(result.sort()).toStrictEqual(expected.sort());
}, 5000);

test('find all files in test fixture ignoring *.txt', async () => {
// six levels deep to ensure node_modules is tested
const { files: result } = await find(testFixture, ['*.txt'], [], 6);
const expected = [
path.join(testFixture, 'maven', 'pom.xml'),
path.join(testFixture, 'mvn', 'pom.xml'),
];
expect(result.sort()).toStrictEqual(expected.sort());
}, 5000);

test('find all files in test fixture by filtering for *.xml', async () => {
// six levels deep to ensure node_modules is tested
const { files: result } = await find(testFixture, [], ['*.xml'], 6);
const expected = [
path.join(testFixture, 'maven', 'pom.xml'),
path.join(testFixture, 'mvn', 'pom.xml'),
];
expect(result.sort()).toStrictEqual(expected.sort());
}, 5000);

test('find all files in test fixture but filter out specific path for mvn', async () => {
const { files: result } = await find(testFixture, ['**/mvn/**.xml'], [], 6);
const expected = [
path.join(testFixture, 'maven', 'pom.xml'),
path.join(testFixture, 'maven', 'test.txt'),
path.join(testFixture, 'mvn', 'test.txt'),
];
expect(result.sort()).toStrictEqual(expected.sort());
}, 5000);

test.todo('Test **/folder/*.txt');

test('find pom.xml files in test fixture', async () => {
const { files: result } = await find(testFixture, [], ['pom.xml']);
const expected = [
path.join(testFixture, 'maven', 'pom.xml'),
path.join(testFixture, 'mvn', 'pom.xml'),
];
expect(result.sort()).toStrictEqual(expected);
});
Empty file.
Empty file.
Empty file.
Empty file.

0 comments on commit 476e8e4

Please sign in to comment.