-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
- Loading branch information
There are no files selected for viewing
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,30 +1,136 @@ | ||
import * as fs from "fs"; | ||
import * as path from "path"; | ||
//import { EXIT, visitParents } from 'unist-util-visit-parents' | ||
import { visit, EXIT } from 'unist-util-visit' | ||
import path from 'path' | ||
import { remark } from 'remark' | ||
import remarkGfm from "remark-gfm"; | ||
import remarkMdx from "remark-mdx"; | ||
|
||
|
||
function walkDir(dirPath, callback, ...callbackArgs) { | ||
fs.readdirSync(dirPath).forEach((file) => { | ||
const filePath = path.join(dirPath, file); | ||
async function walkDir(dir, callback, ...callbackArgs) { | ||
let files = fs.readdirSync(dir); | ||
|
||
for (const file of files) { | ||
const filePath = path.join(dir, file); | ||
const stat = fs.statSync(filePath); | ||
|
||
if (stat.isDirectory()) { | ||
walkDir(filePath, callback, ...callbackArgs); | ||
} else { | ||
callback(filePath, ...callbackArgs); | ||
await walkDir(filePath, callback, ...callbackArgs); | ||
} else if (stat.isFile() && !filePath.toLowerCase().includes(".ds_store")) { | ||
await callback(filePath, ...callbackArgs); | ||
} | ||
}); | ||
} | ||
} | ||
|
||
function readFileAndAddContentToFIle(filePath, contentHolder) { | ||
async function readFileProcessItAndAddContentToFile(filePath, contentHolder) { | ||
const fileContent = fs.readFileSync(filePath, 'utf8') | ||
contentHolder.content += fileContent; | ||
const processedFileContent = await convertMdxToMd(fileContent, filePath) | ||
contentHolder.content += processedFileContent; | ||
} | ||
|
||
const pathsNames = [["", "all"], ["komodo-defi-framework", "komodo-defi-framework"], ["historical", "historical"], ["smart-chains", "smart-chains"], ["antara", "antara"], ["start-here", "start-here"], ["komodo-defi-framework/api", "komodefi-api/all-api"], ["komodo-defi-framework/api/legacy", "komodefi-api/legacy-api"], ["komodo-defi-framework/api/v20", "komodefi-api/v20-api"], ["komodo-defi-framework/api/v20-dev", "komodefi-api/v20-dev-api"]] | ||
|
||
for (let index = 0; index < pathsNames.length; index++) { | ||
const element = pathsNames[index]; | ||
let contentHolder = { content: "" }; | ||
walkDir(`./src/pages/${element[0]}`, readFileAndAddContentToFIle, contentHolder) | ||
await walkDir(`./src/pages/${element[0]}`, readFileProcessItAndAddContentToFile, contentHolder) | ||
fs.writeFileSync(`./data-for-gpts/${element[1]}-content.txt`, contentHolder.content) | ||
} | ||
|
||
async function convertMdxToMd(fileContent, filePath) { | ||
try { | ||
const mdContent = await remark() | ||
.use(remarkGfm) | ||
.use(remarkMdx) | ||
.use(() => (tree) => { | ||
|
||
visit(tree, 'mdxJsxFlowElement', (node, index, parent) => { | ||
|
||
if (node.name === 'BulletPoints') { | ||
const titleProp = node.attributes.find(attr => attr.name === 'title'); | ||
const descProp = node.attributes.find(attr => attr.name === 'desc'); | ||
|
||
const newContent = []; | ||
|
||
if (titleProp && titleProp.value) { | ||
newContent.push({ | ||
type: 'heading', | ||
depth: 3, | ||
children: [{ type: 'text', value: titleProp.value }] | ||
}); | ||
} | ||
|
||
|
||
if (descProp && descProp.value && descProp.value.type === 'mdxJsxAttributeValueExpression') { | ||
try { | ||
const markdown = convertHtmlToMd(descProp.value.value); | ||
newContent.push({ | ||
type: 'paragraph', | ||
children: [{ type: 'text', value: markdown }] | ||
}); | ||
} catch (error) { | ||
console.error(error) | ||
throw new Error(`Error parsing descProp in file: ${filePath}`, error); | ||
} | ||
} | ||
|
||
if (newContent.length > 0) { | ||
parent.children.splice(index, 1, ...newContent); | ||
return [visit.SKIP, index]; | ||
} else { | ||
parent.children.splice(index, 1); | ||
return [visit.SKIP, index]; | ||
} | ||
} | ||
|
||
if (node.children && node.children.length > 0) { | ||
parent.children.splice(index, 1, ...node.children); | ||
return [visit.SKIP, index]; | ||
} else { | ||
parent.children.splice(index, 1); | ||
return [visit.SKIP, index]; | ||
} | ||
}); | ||
|
||
visit(tree, 'mdxJsxTextElement', (node, index, parent) => { | ||
if (node.children && node.children.length > 0) { | ||
parent.children.splice(index, 1, ...node.children); | ||
return [visit.SKIP, index]; | ||
} else { | ||
parent.children.splice(index, 1); | ||
return [visit.SKIP, index]; | ||
} | ||
}); | ||
|
||
|
||
|
||
}) | ||
.process(fileContent); | ||
|
||
return mdContent | ||
} catch (error) { | ||
if (error) { | ||
throw new Error(`Error in file: ${filePath} \n ${error}`); | ||
}; | ||
} | ||
} | ||
|
||
function convertHtmlToMd(html) { | ||
// Remove leading/trailing whitespace and newlines | ||
html = html.trim(); | ||
|
||
// Replace <ul> and </ul> tags | ||
html = html.replace(/<\/?ul>/g, ''); | ||
|
||
// Replace <li> tags with Markdown list items | ||
html = html.replace(/<li>/g, '- '); | ||
|
||
// Remove </li> tags | ||
html = html.replace(/<\/li>/g, ''); | ||
|
||
// Trim each line and remove empty lines | ||
const lines = html.split('\n').map(line => line.trim()).filter(line => line !== ''); | ||
|
||
// Join the lines with newline characters | ||
return lines.join('\n'); | ||
} |