From 6712e307ad4588799b8c8460e3658b0d27bfda73 Mon Sep 17 00:00:00 2001 From: mohamed yahia Date: Tue, 14 Nov 2023 00:15:02 +0200 Subject: [PATCH] pathToUrlResolver / utils functions / --- src/lib/indexFolderToObjects.ts | 52 ++++++++++++++++-------- src/lib/markdowndb.ts | 24 ++++------- src/lib/types/FileObject.ts | 7 ++-- src/lib/types/schemaTypes.ts | 2 +- src/utils/defaultFilePathToUrl.ts | 8 ++++ src/utils/extractFileSchemeFromObject.ts | 12 ++++++ src/utils/index.ts | 4 +- 7 files changed, 69 insertions(+), 40 deletions(-) create mode 100644 src/utils/defaultFilePathToUrl.ts create mode 100644 src/utils/extractFileSchemeFromObject.ts diff --git a/src/lib/indexFolderToObjects.ts b/src/lib/indexFolderToObjects.ts index d61d465..6290196 100644 --- a/src/lib/indexFolderToObjects.ts +++ b/src/lib/indexFolderToObjects.ts @@ -1,48 +1,55 @@ -import { WikiLink, getUniqueValues, recursiveWalkDir } from "../utils/index.js"; -import { markdownToObject } from "./markdownToObject.js"; +import { getUniqueValues, recursiveWalkDir } from "../utils/index.js"; +import type { WikiLink } from "../utils/index.js"; +import { extractFileSchemeFromObject } from "../utils/extractFileSchemeFromObject.js"; +import { readLocalMarkdownFileToObject } from "./readLocalMarkdownFileToObject.js"; import { File, FileTag, Link, Tag } from "./types/schemaTypes.js"; export function indexFolderToObjects( folderPath: string, - ignorePatterns?: RegExp[], - pathToUrlResolver?: (filePath: string) => string + pathToUrlResolver: (filePath: string) => string, + ignorePatterns?: RegExp[] ) { const filePathsToIndex = recursiveWalkDir(folderPath); const files: File[] = []; - const tags: Tag[] = []; + const tags: string[] = []; const fileTags: FileTag[] = []; const links: Link[] = []; - const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) => { - return !( - ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath)) - ); - }); + const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) => + shouldIncludeFile(filePath, ignorePatterns) + ); for (const filePath of filteredFilePathsToIndex) { - const fileObject = markdownToObject(folderPath, filePath, filePathsToIndex); + const fileObject = readLocalMarkdownFileToObject( + folderPath, + filePath, + filePathsToIndex, + pathToUrlResolver + ); + + const file = extractFileSchemeFromObject(fileObject); + files.push(file); - files.push(fileObject.file); tags.push(...fileObject.tags); const fileTagsToInsert = fileObject.tags.map((tag) => ({ - tag: tag.name, - file: fileObject.file._id, + tag: tag, + file: fileObject._id, })); - const uniqueFileTags = getUniqueValues(fileTagsToInsert); - fileTags.push(...uniqueFileTags); + fileTags.push(...fileTagsToInsert); const linksToInsert: Link[] = processWikiLinks( fileObject.links, - fileObject.file._id, + fileObject._id, filePathsToIndex ); links.push(...linksToInsert); } const uniqueTags = getUniqueValues(tags); + const TagsToInsert = uniqueTags.map((tag) => ({ name: tag })); return { files: files, - tags: uniqueTags, + tags: TagsToInsert, fileTags: fileTags, links: links, }; @@ -61,3 +68,12 @@ function processWikiLinks( })) .filter((link) => link.to !== undefined); } + +function shouldIncludeFile( + filePath: string, + ignorePatterns?: RegExp[] +): boolean { + return !( + ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath)) + ); +} diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts index a08ef56..8e9eb87 100644 --- a/src/lib/markdowndb.ts +++ b/src/lib/markdowndb.ts @@ -2,17 +2,9 @@ import knex, { Knex } from "knex"; import { MddbFile, MddbTag, MddbFileTag, MddbLink } from "./schema.js"; import { DbQueryManager } from "./DbQueryManager.js"; -import { FilesQuery, LinkQuery } from "./types/DbQueryTypes.js"; +import type { FilesQuery, LinkQuery } from "./types/DbQueryTypes.js"; import { indexFolderToObjects } from "./indexFolderToObjects.js"; - -export const defaultFilePathToUrl = (filePath: string) => { - let url = filePath - .replace(/\.(mdx|md)/, "") - .replace(/\\/g, "/") // replace windows backslash with forward slash - .replace(/(\/)?index$/, ""); // remove index from the end of the permalink - url = url.length > 0 ? url : "/"; // for home page - return encodeURI(url); -}; +import { defaultFilePathToUrl } from "../utils/defaultFilePathToUrl.js"; export class MarkdownDB { config: Knex.Config; @@ -42,14 +34,14 @@ export class MarkdownDB { await resetDatabaseTables(this.db); const { files, tags, fileTags, links } = indexFolderToObjects( folderPath, - ignorePatterns, - pathToUrlResolver + pathToUrlResolver, + ignorePatterns ); - MddbFile.batchInsert(this.db, files); - MddbTag.batchInsert(this.db, tags); - MddbFileTag.batchInsert(this.db, fileTags); - MddbLink.batchInsert(this.db, links); + await MddbFile.batchInsert(this.db, files); + await MddbTag.batchInsert(this.db, tags); + await MddbFileTag.batchInsert(this.db, fileTags); + await MddbLink.batchInsert(this.db, links); } async getFileById(id: string): Promise { diff --git a/src/lib/types/FileObject.ts b/src/lib/types/FileObject.ts index 3feefce..ab28a59 100644 --- a/src/lib/types/FileObject.ts +++ b/src/lib/types/FileObject.ts @@ -1,8 +1,7 @@ import { WikiLink } from "../../utils"; -import { File, Tag } from "./schemaTypes"; +import { File } from "./schemaTypes"; -export interface FileObject { - file: File; - tags: Tag[]; +export interface FileObject extends File { + tags: string[]; links: WikiLink[]; } diff --git a/src/lib/types/schemaTypes.ts b/src/lib/types/schemaTypes.ts index f36a22c..5595be8 100644 --- a/src/lib/types/schemaTypes.ts +++ b/src/lib/types/schemaTypes.ts @@ -31,4 +31,4 @@ export enum Table { Tags = "tags", FileTags = "file_tags", Links = "links", -} \ No newline at end of file +} diff --git a/src/utils/defaultFilePathToUrl.ts b/src/utils/defaultFilePathToUrl.ts new file mode 100644 index 0000000..8e68990 --- /dev/null +++ b/src/utils/defaultFilePathToUrl.ts @@ -0,0 +1,8 @@ +export const defaultFilePathToUrl = (filePath: string) => { + let url = filePath + .replace(/\.(mdx|md)/, "") + .replace(/\\/g, "/") // replace windows backslash with forward slash + .replace(/(\/)?index$/, ""); // remove index from the end of the permalink + url = url.length > 0 ? url : "/"; // for home page + return encodeURI(url); +}; diff --git a/src/utils/extractFileSchemeFromObject.ts b/src/utils/extractFileSchemeFromObject.ts new file mode 100644 index 0000000..3e4675c --- /dev/null +++ b/src/utils/extractFileSchemeFromObject.ts @@ -0,0 +1,12 @@ +import { FileObject } from "../lib/types/FileObject.js"; + +export function extractFileSchemeFromObject(fileObject: FileObject) { + return { + _id: fileObject._id, + file_path: fileObject.file_path, + extension: fileObject.extension, + url_path: fileObject.url_path, + filetype: fileObject.filetype, + metadata: fileObject.metadata, + }; +} diff --git a/src/utils/index.ts b/src/utils/index.ts index dbe73c3..a58b933 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,6 +1,8 @@ export { recursiveWalkDir } from "./recursiveWalkDir.js"; export { extractWikiLinks, WikiLink } from "./extractWikiLinks.js"; -export { parseFile } from "./parseFile.js"; +export { parseMarkdownContent } from "./parseMarkdownContent.js"; export { getUniqueValues } from "./getUniqueValues.js"; export { generateFileIdFromPath } from "./generateFileIdFromPath.js"; export { getFileExtensionFromPath } from "./getFileExtensionFromPath.js"; +export { defaultFilePathToUrl } from "./defaultFilePathToUrl.js"; +export { extractFileSchemeFromObject } from "./extractFileSchemeFromObject.js";