From c496199373745c066d82875700fcedd1e476ac3e Mon Sep 17 00:00:00 2001
From: mohamed yahia <salemmohamed780@gmail.com>
Date: Wed, 20 Dec 2023 01:00:41 +0200
Subject: [PATCH] Add configuration with include/exclude options (#97)
* Add configuration with include/exclude options
* Create changeset
* Make config file path optional
---
.changeset/rude-pants-add.md | 5 ++++
markdowndb.config.js | 8 +++++
package-lock.json | 20 +++++++++++--
package.json | 2 ++
src/bin/index.js | 25 +++++++++++-----
src/lib/CustomConfig.ts | 2 ++
src/lib/indexFolder.ts | 57 +++++++++++++++++++++++++++++++-----
src/lib/loadConfig.ts | 18 ++++++++++++
src/lib/markdowndb.ts | 19 +++++++++---
9 files changed, 134 insertions(+), 22 deletions(-)
create mode 100644 .changeset/rude-pants-add.md
create mode 100644 markdowndb.config.js
create mode 100644 src/lib/loadConfig.ts
diff --git a/.changeset/rude-pants-add.md b/.changeset/rude-pants-add.md
new file mode 100644
index 0000000..70c0b8d
--- /dev/null
+++ b/.changeset/rude-pants-add.md
@@ -0,0 +1,5 @@
+---
+"mddb": minor
+---
+
+- Add configuration with include/exclude options
diff --git a/markdowndb.config.js b/markdowndb.config.js
new file mode 100644
index 0000000..cd8dfb2
--- /dev/null
+++ b/markdowndb.config.js
@@ -0,0 +1,8 @@
+export default {
+ computedFields: [], // Array of functions to computed fields
+ schemas: {
+ // Add zod schemas
+ },
+ include: [], // Pattern for files to be included
+ exclude: [] // Patten for files to be excluded
+};
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 285faf0..b8da53b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,18 +1,20 @@
{
"name": "mddb",
- "version": "0.7.0",
+ "version": "0.8.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "mddb",
- "version": "0.7.0",
+ "version": "0.8.0",
"license": "MIT",
"dependencies": {
"@portaljs/remark-wiki-link": "^1.0.4",
+ "@types/micromatch": "^4.0.6",
"chokidar": "^3.5.3",
"gray-matter": "^4.0.3",
"knex": "^2.4.2",
+ "micromatch": "^4.0.5",
"react-markdown": "^9.0.1",
"remark-gfm": "^3.0.1",
"remark-parse": "^10.0.1",
@@ -2156,6 +2158,11 @@
"@babel/types": "^7.3.0"
}
},
+ "node_modules/@types/braces": {
+ "version": "3.0.4",
+ "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.4.tgz",
+ "integrity": "sha512-0WR3b8eaISjEW7RpZnclONaLFDf7buaowRHdqLp4vLj54AsSAYWfh3DRbfiYJY9XDxMgx1B4sE1Afw2PGpuHOA=="
+ },
"node_modules/@types/debug": {
"version": "4.1.7",
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz",
@@ -2238,6 +2245,14 @@
"@types/unist": "*"
}
},
+ "node_modules/@types/micromatch": {
+ "version": "4.0.6",
+ "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.6.tgz",
+ "integrity": "sha512-2eulCHWqjEpk9/vyic4tBhI8a9qQEl6DaK2n/sF7TweX9YESlypgKyhXMDGt4DAOy/jhLPvVrZc8pTDAMsplJA==",
+ "dependencies": {
+ "@types/braces": "*"
+ }
+ },
"node_modules/@types/minimist": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz",
@@ -7685,7 +7700,6 @@
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
"integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
- "dev": true,
"dependencies": {
"braces": "^3.0.2",
"picomatch": "^2.3.1"
diff --git a/package.json b/package.json
index 2881a47..549272c 100644
--- a/package.json
+++ b/package.json
@@ -45,9 +45,11 @@
"types": "./dist/src/index.d.ts",
"dependencies": {
"@portaljs/remark-wiki-link": "^1.0.4",
+ "@types/micromatch": "^4.0.6",
"chokidar": "^3.5.3",
"gray-matter": "^4.0.3",
"knex": "^2.4.2",
+ "micromatch": "^4.0.5",
"react-markdown": "^9.0.1",
"remark-gfm": "^3.0.1",
"remark-parse": "^10.0.1",
diff --git a/src/bin/index.js b/src/bin/index.js
index 6e71d78..238214f 100755
--- a/src/bin/index.js
+++ b/src/bin/index.js
@@ -1,17 +1,27 @@
#!/usr/bin/env node
-
import { MarkdownDB } from "../lib/markdowndb.js";
// TODO get these from markdowndb.config.js or something
const dbPath = "markdown.db";
const ignorePatterns = [/Excalidraw/, /\.obsidian/, /DS_Store/];
-const [contentPath, watchFlag] = process.argv.slice(2);
-if (!contentPath) {
- throw new Error("Invalid/Missing path to markdown content folder");
+let watchFlag;
+const args = process.argv.slice(2);
+
+// Check for the watch flag and its position
+const watchIndex = args.indexOf('--watch');
+if (watchIndex !== -1) {
+ watchFlag = args[watchIndex];
+ args.splice(watchIndex, 1); // Remove the watch flag from the array
}
-const watchEnabled = watchFlag && watchFlag === "--watch";
+// Assign values to contentPath and configFilePath based on their positions
+const [contentPath, configFilePath] = args;
+
+if (!contentPath) {
+ console.error('Invalid/Missing path to markdown content folder');
+ process.exit(1);
+}
const client = new MarkdownDB({
client: "sqlite3",
@@ -25,9 +35,10 @@ await client.init();
await client.indexFolder({
folderPath: contentPath,
ignorePatterns: ignorePatterns,
- watch: watchEnabled,
+ watch: watchFlag,
+ configFilePath: configFilePath
});
-if (!watchEnabled) {
+if (!watchFlag) {
process.exit();
}
diff --git a/src/lib/CustomConfig.ts b/src/lib/CustomConfig.ts
index 32ca29d..eab3cf3 100644
--- a/src/lib/CustomConfig.ts
+++ b/src/lib/CustomConfig.ts
@@ -8,4 +8,6 @@ type Schemas = { [index: string]: ZodObject<any> };
export interface CustomConfig {
computedFields?: ComputedFields;
schemas?: Schemas;
+ include?: string[];
+ exclude?: string[];
}
diff --git a/src/lib/indexFolder.ts b/src/lib/indexFolder.ts
index c017df8..a909d58 100644
--- a/src/lib/indexFolder.ts
+++ b/src/lib/indexFolder.ts
@@ -2,6 +2,7 @@ import { ZodError } from "zod";
import { CustomConfig } from "./CustomConfig.js";
import { FileInfo, processFile } from "./process.js";
import { recursiveWalkDir } from "./recursiveWalkDir.js";
+import micromatch from "micromatch";
export function indexFolder(
folderPath: string,
@@ -11,7 +12,12 @@ export function indexFolder(
) {
const filePathsToIndex = recursiveWalkDir(folderPath);
const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) =>
- shouldIncludeFile(filePath, ignorePatterns)
+ shouldIncludeFile({
+ filePath,
+ ignorePatterns,
+ includeGlob: config.include,
+ excludeGlob: config.exclude,
+ })
);
const files: FileInfo[] = [];
const computedFields = config.computedFields || [];
@@ -56,11 +62,46 @@ export function indexFolder(
return files;
}
-export function shouldIncludeFile(
- filePath: string,
- ignorePatterns?: RegExp[]
-): boolean {
- return !(
- ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
- );
+export function shouldIncludeFile({
+ filePath,
+ ignorePatterns,
+ includeGlob,
+ excludeGlob,
+}: {
+ filePath: string;
+ ignorePatterns?: RegExp[];
+ includeGlob?: string[];
+ excludeGlob?: string[];
+}): boolean {
+ const normalizedFilePath = filePath.replace(/\\/g, "/");
+
+ if (
+ ignorePatterns &&
+ ignorePatterns.some((pattern) => pattern.test(normalizedFilePath))
+ ) {
+ return false;
+ }
+
+ // Check if the file should be included based on includeGlob
+ if (
+ includeGlob &&
+ includeGlob.length > 0 &&
+ !includeGlob.some((pattern) =>
+ micromatch.isMatch(normalizedFilePath, pattern)
+ )
+ ) {
+ return false;
+ }
+
+ // Check if the file should be excluded based on excludeGlob
+ if (
+ excludeGlob &&
+ excludeGlob.some((pattern) =>
+ micromatch.isMatch(normalizedFilePath, pattern)
+ )
+ ) {
+ return false;
+ }
+
+ return true;
}
diff --git a/src/lib/loadConfig.ts b/src/lib/loadConfig.ts
new file mode 100644
index 0000000..a084bd9
--- /dev/null
+++ b/src/lib/loadConfig.ts
@@ -0,0 +1,18 @@
+import * as path from "path";
+
+export async function loadConfig(configFilePath?: string) {
+ const normalizedPath = path.resolve(configFilePath || "markdowndb.config.js");
+ const fileUrl = new URL(`file://${normalizedPath}`);
+
+ try {
+ // Import the module using the file URL
+ const configModule = await import(fileUrl.href);
+ return configModule.default;
+ } catch (error) {
+ if (configFilePath) {
+ throw new Error(
+ `Error loading configuration file from ${normalizedPath}`
+ );
+ }
+ }
+}
diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts
index 1ec25ec..c017426 100644
--- a/src/lib/markdowndb.ts
+++ b/src/lib/markdowndb.ts
@@ -17,6 +17,7 @@ import { CustomConfig } from "./CustomConfig.js";
import { FileInfo, processFile } from "./process.js";
import chokidar from "chokidar";
import { recursiveWalkDir } from "./recursiveWalkDir.js";
+import { loadConfig } from "./loadConfig.js";
const defaultFilePathToUrl = (filePath: string) => {
let url = filePath
@@ -77,19 +78,22 @@ export class MarkdownDB {
// TODO support glob patterns
ignorePatterns = [],
pathToUrlResolver = defaultFilePathToUrl,
- customConfig = {},
+ customConfig,
watch = false,
+ configFilePath,
}: {
folderPath: string;
ignorePatterns?: RegExp[];
pathToUrlResolver?: (filePath: string) => string;
customConfig?: CustomConfig;
watch?: boolean;
+ configFilePath?: string;
}) {
+ const config = customConfig || (await loadConfig(configFilePath)) || {};
const fileObjects = indexFolder(
folderPath,
pathToUrlResolver,
- customConfig,
+ config,
ignorePatterns
);
await this.saveDataToDisk(fileObjects);
@@ -100,10 +104,17 @@ export class MarkdownDB {
});
const filePathsToIndex = recursiveWalkDir(folderPath);
- const computedFields = customConfig.computedFields || [];
+ const computedFields = config.computedFields || [];
const handleFileEvent = (event: string, filePath: string) => {
- if (!shouldIncludeFile(filePath, ignorePatterns)) {
+ if (
+ !shouldIncludeFile({
+ filePath,
+ ignorePatterns,
+ includeGlob: config.include,
+ excludeGlob: config.exclude,
+ })
+ ) {
return;
}