pathToUrlResolver / utils functions /

src/lib/indexFolderToObjects.ts

@@ -1,48 +1,55 @@
-import { WikiLink, getUniqueValues, recursiveWalkDir } from "../utils/index.js";
-import { markdownToObject } from "./markdownToObject.js";
+import { getUniqueValues, recursiveWalkDir } from "../utils/index.js";
+import type { WikiLink } from "../utils/index.js";
+import { extractFileSchemeFromObject } from "../utils/extractFileSchemeFromObject.js";
+import { readLocalMarkdownFileToObject } from "./readLocalMarkdownFileToObject.js";
 import { File, FileTag, Link, Tag } from "./types/schemaTypes.js";
 
 export function indexFolderToObjects(
   folderPath: string,
-  ignorePatterns?: RegExp[],
-  pathToUrlResolver?: (filePath: string) => string
+  pathToUrlResolver: (filePath: string) => string,
+  ignorePatterns?: RegExp[]
 ) {
   const filePathsToIndex = recursiveWalkDir(folderPath);
   const files: File[] = [];
-  const tags: Tag[] = [];
+  const tags: string[] = [];
   const fileTags: FileTag[] = [];
   const links: Link[] = [];
-  const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) => {
-    return !(
-      ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
-    );
-  });
+  const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) =>
+    shouldIncludeFile(filePath, ignorePatterns)
+  );
 
   for (const filePath of filteredFilePathsToIndex) {
-    const fileObject = markdownToObject(folderPath, filePath, filePathsToIndex);
+    const fileObject = readLocalMarkdownFileToObject(
+      folderPath,
+      filePath,
+      filePathsToIndex,
+      pathToUrlResolver
+    );
+
+    const file = extractFileSchemeFromObject(fileObject);
+    files.push(file);
 
-    files.push(fileObject.file);
     tags.push(...fileObject.tags);
 
     const fileTagsToInsert = fileObject.tags.map((tag) => ({
-      tag: tag.name,
-      file: fileObject.file._id,
+      tag: tag,
+      file: fileObject._id,
     }));
-    const uniqueFileTags = getUniqueValues(fileTagsToInsert);
-    fileTags.push(...uniqueFileTags);
+    fileTags.push(...fileTagsToInsert);
 
     const linksToInsert: Link[] = processWikiLinks(
       fileObject.links,
-      fileObject.file._id,
+      fileObject._id,
       filePathsToIndex
     );
     links.push(...linksToInsert);
   }
 
   const uniqueTags = getUniqueValues(tags);
+  const TagsToInsert = uniqueTags.map((tag) => ({ name: tag }));
   return {
     files: files,
-    tags: uniqueTags,
+    tags: TagsToInsert,
     fileTags: fileTags,
     links: links,
   };
@@ -61,3 +68,12 @@ function processWikiLinks(
     }))
     .filter((link) => link.to !== undefined);
 }
+
+function shouldIncludeFile(
+  filePath: string,
+  ignorePatterns?: RegExp[]
+): boolean {
+  return !(
+    ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
+  );
+}

src/lib/markdowndb.ts

@@ -2,17 +2,9 @@ import knex, { Knex } from "knex";
 
 import { MddbFile, MddbTag, MddbFileTag, MddbLink } from "./schema.js";
 import { DbQueryManager } from "./DbQueryManager.js";
-import { FilesQuery, LinkQuery } from "./types/DbQueryTypes.js";
+import type { FilesQuery, LinkQuery } from "./types/DbQueryTypes.js";
 import { indexFolderToObjects } from "./indexFolderToObjects.js";
-
-export const defaultFilePathToUrl = (filePath: string) => {
-  let url = filePath
-    .replace(/\.(mdx|md)/, "")
-    .replace(/\\/g, "/") // replace windows backslash with forward slash
-    .replace(/(\/)?index$/, ""); // remove index from the end of the permalink
-  url = url.length > 0 ? url : "/"; // for home page
-  return encodeURI(url);
-};
+import { defaultFilePathToUrl } from "../utils/defaultFilePathToUrl.js";
 
 export class MarkdownDB {
   config: Knex.Config;
@@ -42,14 +34,14 @@ export class MarkdownDB {
     await resetDatabaseTables(this.db);
     const { files, tags, fileTags, links } = indexFolderToObjects(
       folderPath,
-      ignorePatterns,
-      pathToUrlResolver
+      pathToUrlResolver,
+      ignorePatterns
     );
 
-    MddbFile.batchInsert(this.db, files);
-    MddbTag.batchInsert(this.db, tags);
-    MddbFileTag.batchInsert(this.db, fileTags);
-    MddbLink.batchInsert(this.db, links);
+    await MddbFile.batchInsert(this.db, files);
+    await MddbTag.batchInsert(this.db, tags);
+    await MddbFileTag.batchInsert(this.db, fileTags);
+    await MddbLink.batchInsert(this.db, links);
   }
 
   async getFileById(id: string): Promise<MddbFile | null> {

src/lib/types/FileObject.ts

@@ -1,8 +1,7 @@
 import { WikiLink } from "../../utils";
-import { File, Tag } from "./schemaTypes";
+import { File } from "./schemaTypes";
 
-export interface FileObject {
-  file: File;
-  tags: Tag[];
+export interface FileObject extends File {
+  tags: string[];
   links: WikiLink[];
 }

src/lib/types/schemaTypes.ts

@@ -31,4 +31,4 @@ export enum Table {
   Tags = "tags",
   FileTags = "file_tags",
   Links = "links",
-}
+}

src/utils/defaultFilePathToUrl.ts

@@ -0,0 +1,8 @@
+export const defaultFilePathToUrl = (filePath: string) => {
+  let url = filePath
+    .replace(/\.(mdx|md)/, "")
+    .replace(/\\/g, "/") // replace windows backslash with forward slash
+    .replace(/(\/)?index$/, ""); // remove index from the end of the permalink
+  url = url.length > 0 ? url : "/"; // for home page
+  return encodeURI(url);
+};

src/utils/extractFileSchemeFromObject.ts

@@ -0,0 +1,12 @@
+import { FileObject } from "../lib/types/FileObject.js";
+
+export function extractFileSchemeFromObject(fileObject: FileObject) {
+  return {
+    _id: fileObject._id,
+    file_path: fileObject.file_path,
+    extension: fileObject.extension,
+    url_path: fileObject.url_path,
+    filetype: fileObject.filetype,
+    metadata: fileObject.metadata,
+  };
+}

src/utils/index.ts

@@ -1,6 +1,8 @@
 export { recursiveWalkDir } from "./recursiveWalkDir.js";
 export { extractWikiLinks, WikiLink } from "./extractWikiLinks.js";
-export { parseFile } from "./parseFile.js";
+export { parseMarkdownContent } from "./parseMarkdownContent.js";
 export { getUniqueValues } from "./getUniqueValues.js";
 export { generateFileIdFromPath } from "./generateFileIdFromPath.js";
 export { getFileExtensionFromPath } from "./getFileExtensionFromPath.js";
+export { defaultFilePathToUrl } from "./defaultFilePathToUrl.js";
+export { extractFileSchemeFromObject } from "./extractFileSchemeFromObject.js";