From 95598386d41abf8199885c424004ee16e3df0bfa Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Wed, 11 Sep 2024 15:37:21 -0400 Subject: [PATCH] feat: script to validate all markdown validation errors --- packages/ui/app/src/index.ts | 2 +- packages/ui/app/src/mdx/bundler.ts | 28 ++- .../ui/app/src/mdx/bundlers/mdx-bundler.ts | 170 +++++++++--------- .../app/src/mdx/bundlers/next-mdx-remote.ts | 35 ++-- .../pages/api/fern-docs/validate/mdx/v1.ts | 153 ++++++++++++++++ 5 files changed, 272 insertions(+), 116 deletions(-) create mode 100644 packages/ui/docs-bundle/src/pages/api/fern-docs/validate/mdx/v1.ts diff --git a/packages/ui/app/src/index.ts b/packages/ui/app/src/index.ts index d22317d056..60dca4cb66 100644 --- a/packages/ui/app/src/index.ts +++ b/packages/ui/app/src/index.ts @@ -4,7 +4,7 @@ export type { DocsProps, FeatureFlags } from "./atoms"; export * from "./docs/DocsPage"; export * from "./docs/NextApp"; export { getApiRouteSupplier } from "./hooks/useApiRoute"; -export { serializeMdx, setMdxBundler } from "./mdx/bundler"; +export { serializeMdx, setMdxBundler, unsafeSerializeMdx } from "./mdx/bundler"; export { getFrontmatter } from "./mdx/frontmatter"; export { Stream } from "./playground/Stream"; export { ProxyRequestSchema } from "./playground/types"; diff --git a/packages/ui/app/src/mdx/bundler.ts b/packages/ui/app/src/mdx/bundler.ts index f5cf5e7b3a..9c630473f0 100644 --- a/packages/ui/app/src/mdx/bundler.ts +++ b/packages/ui/app/src/mdx/bundler.ts @@ -1,3 +1,4 @@ +import { captureSentryError } from "../analytics/sentry"; import { serializeMdx as defaultSerializeMdx } from "./bundlers/next-mdx-remote"; import type { BundledMDX, FernSerializeMdxOptions, SerializeMdxFunc } from "./types"; @@ -11,12 +12,12 @@ export function setMdxBundler(engine: SerializeMdxFunc): void { currentEngine = engine; } -export async function serializeMdx(content: string, options?: FernSerializeMdxOptions): Promise; -export async function serializeMdx( +export async function unsafeSerializeMdx(content: string, options?: FernSerializeMdxOptions): Promise; +export async function unsafeSerializeMdx( content: string | undefined, options?: FernSerializeMdxOptions, ): Promise; -export async function serializeMdx( +export async function unsafeSerializeMdx( content: string | undefined, options: FernSerializeMdxOptions = {}, ): Promise { @@ -27,3 +28,24 @@ export async function serializeMdx( const bundler = await getMdxBundler(); return bundler(content, options); } + +export async function serializeMdx(content: string, options?: FernSerializeMdxOptions): Promise; +export async function serializeMdx( + content: string | undefined, + options?: FernSerializeMdxOptions, +): Promise; +export async function serializeMdx( + content: string | undefined, + options: FernSerializeMdxOptions = {}, +): Promise { + try { + return unsafeSerializeMdx(content, options); + } catch (e) { + captureSentryError(e, { + context: "MDX", + errorSource: "maybeSerializeMdxContent", + errorDescription: "Failed to serialize MDX content", + }); + } + return content; +} diff --git a/packages/ui/app/src/mdx/bundlers/mdx-bundler.ts b/packages/ui/app/src/mdx/bundlers/mdx-bundler.ts index 2c32a84195..df9278df3d 100644 --- a/packages/ui/app/src/mdx/bundlers/mdx-bundler.ts +++ b/packages/ui/app/src/mdx/bundlers/mdx-bundler.ts @@ -55,97 +55,91 @@ export async function serializeMdx( } } - try { - const bundled = await bundleMDX({ - source: content, - files: mapKeys(files ?? {}, (_file, filename) => { - if (cwd != null) { - return path.relative(cwd, filename); - } - return filename; - }), - - mdxOptions: (o: Options, matter) => { - o.remarkRehypeOptions = { - ...o.remarkRehypeOptions, - ...options, - handlers: { - ...o.remarkRehypeOptions?.handlers, - heading: customHeadingHandler, - ...options?.remarkRehypeOptions?.handlers, - }, + const bundled = await bundleMDX({ + source: content, + files: mapKeys(files ?? {}, (_file, filename) => { + if (cwd != null) { + return path.relative(cwd, filename); + } + return filename; + }), + + mdxOptions: (o: Options, matter) => { + o.remarkRehypeOptions = { + ...o.remarkRehypeOptions, + ...options, + handlers: { + ...o.remarkRehypeOptions?.handlers, + heading: customHeadingHandler, + ...options?.remarkRehypeOptions?.handlers, + }, + }; + + const remarkPlugins: PluggableList = [ + remarkSqueezeParagraphs, + remarkGfm, + remarkSmartypants, + remarkMath, + remarkGemoji, + ]; + + if (options?.remarkPlugins != null) { + remarkPlugins.push(...options.remarkPlugins); + } + + o.remarkPlugins = [...(o.remarkPlugins ?? []), ...remarkPlugins, ...(options?.remarkPlugins ?? [])]; + + const rehypePlugins: PluggableList = [ + rehypeSqueezeParagraphs, + rehypeSlug, + rehypeKatex, + rehypeFernCode, + rehypeFernComponents, + ]; + + if (options?.rehypePlugins != null) { + rehypePlugins.push(...options.rehypePlugins); + } + + if (frontmatterDefaults != null) { + const opts = { + matter: mergeMatter(matter, frontmatterDefaults), }; + rehypePlugins.push([rehypeFernLayout, opts]); + frontmatter = opts.matter; + } else { + frontmatter = mergeMatter(matter, frontmatter); + } - const remarkPlugins: PluggableList = [ - remarkSqueezeParagraphs, - remarkGfm, - remarkSmartypants, - remarkMath, - remarkGemoji, - ]; - - if (options?.remarkPlugins != null) { - remarkPlugins.push(...options.remarkPlugins); - } - - o.remarkPlugins = [...(o.remarkPlugins ?? []), ...remarkPlugins, ...(options?.remarkPlugins ?? [])]; - - const rehypePlugins: PluggableList = [ - rehypeSqueezeParagraphs, - rehypeSlug, - rehypeKatex, - rehypeFernCode, - rehypeFernComponents, - ]; - - if (options?.rehypePlugins != null) { - rehypePlugins.push(...options.rehypePlugins); - } - - if (frontmatterDefaults != null) { - const opts = { - matter: mergeMatter(matter, frontmatterDefaults), - }; - rehypePlugins.push([rehypeFernLayout, opts]); - frontmatter = opts.matter; - } else { - frontmatter = mergeMatter(matter, frontmatter); - } - - // Always sanitize JSX at the end. - // rehypePlugins.push([rehypeSanitizeJSX, { showError }]); - - o.rehypePlugins = [...(o.rehypePlugins ?? []), ...rehypePlugins, ...(options?.rehypePlugins ?? [])]; - - o.recmaPlugins = [...(o.recmaPlugins ?? []), ...(options?.recmaPlugins ?? [])]; - - o.development = options.development ?? o.development; - - return o; - }, - - esbuildOptions: (o) => { - o.minify = disableMinify ? false : true; - return o; - }, - }); + // Always sanitize JSX at the end. + // rehypePlugins.push([rehypeSanitizeJSX, { showError }]); - if (bundled.errors.length > 0) { - bundled.errors.forEach((error) => { - // eslint-disable-next-line no-console - console.error(error); - }); - } + o.rehypePlugins = [...(o.rehypePlugins ?? []), ...rehypePlugins, ...(options?.rehypePlugins ?? [])]; + + o.recmaPlugins = [...(o.recmaPlugins ?? []), ...(options?.recmaPlugins ?? [])]; + + o.development = options.development ?? o.development; + + return o; + }, - return { - engine: "mdx-bundler", - code: bundled.code, - frontmatter, - errors: bundled.errors, - }; - } catch (e) { - // eslint-disable-next-line no-console - console.error(e); - return content; + esbuildOptions: (o) => { + o.minify = disableMinify ? false : true; + return o; + }, + }); + + if (bundled.errors.length > 0) { + bundled.errors.forEach((error) => { + // eslint-disable-next-line no-console + console.error(error); + }); } + + return { + engine: "mdx-bundler", + code: bundled.code, + frontmatter, + errors: bundled.errors, + }; } diff --git a/packages/ui/app/src/mdx/bundlers/next-mdx-remote.ts b/packages/ui/app/src/mdx/bundlers/next-mdx-remote.ts index a3c716284e..ed6178c19a 100644 --- a/packages/ui/app/src/mdx/bundlers/next-mdx-remote.ts +++ b/packages/ui/app/src/mdx/bundlers/next-mdx-remote.ts @@ -6,7 +6,6 @@ import remarkGfm from "remark-gfm"; import remarkMath from "remark-math"; import remarkSmartypants from "remark-smartypants"; import type { PluggableList } from "unified"; -import { captureSentryError } from "../../analytics/sentry"; import { stringHasMarkdown } from "../common/util"; import { FernDocsFrontmatter } from "../frontmatter"; import { rehypeFernCode } from "../plugins/rehypeFernCode"; @@ -103,28 +102,16 @@ export async function serializeMdx( content = replaceBrokenBrTags(content); - try { - const result = await serialize, FernDocsFrontmatter>(content, { - scope: {}, - mdxOptions: withDefaultMdxOptions(options), - parseFrontmatter: true, - }); - return { - engine: "next-mdx-remote", - code: result.compiledSource, - frontmatter: result.frontmatter, - errors: [], - }; - } catch (e) { - // eslint-disable-next-line no-console - console.error(e); + const result = await serialize, FernDocsFrontmatter>(content, { + scope: {}, + mdxOptions: withDefaultMdxOptions(options), + parseFrontmatter: true, + }); - captureSentryError(e, { - context: "MDX", - errorSource: "maybeSerializeMdxContent", - errorDescription: "Failed to serialize MDX content", - }); - - return content; - } + return { + engine: "next-mdx-remote", + code: result.compiledSource, + frontmatter: result.frontmatter, + errors: [], + }; } diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/validate/mdx/v1.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/validate/mdx/v1.ts new file mode 100644 index 0000000000..5d4ea97524 --- /dev/null +++ b/packages/ui/docs-bundle/src/pages/api/fern-docs/validate/mdx/v1.ts @@ -0,0 +1,153 @@ +/* eslint-disable import/no-internal-modules */ +import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { setMdxBundler, unsafeSerializeMdx } from "@fern-ui/ui"; +import { getAuthEdgeConfig } from "@fern-ui/ui/auth"; +import { getMdxBundler } from "@fern-ui/ui/bundlers"; +import { NextApiHandler, NextApiRequest, NextApiResponse } from "next"; +import { loadWithUrl } from "../../../../../utils/loadWithUrl"; +import { getXFernHostNode } from "../../../../../utils/xFernHost"; +import { getFeatureFlags } from "../../feature-flags"; + +export const config = { + maxDuration: 300, +}; + +interface ValidationError { + markdown: string; + message: string; + slug: string; +} + +const handler: NextApiHandler = async ( + req: NextApiRequest, + res: NextApiResponse, +): Promise => { + try { + // when we call res.revalidate() nextjs uses + // req.headers.host to make the network request + const xFernHost = getXFernHostNode(req, true); + + const authConfig = await getAuthEdgeConfig(xFernHost); + + /** + * If the auth config is basic_token_verification, we don't need to revalidate. + * + * This is because basic_token_verification is a special case where all the routes are protected by a fern_token that + * is generated by the customer, and so all routes use SSR and are not cached. + */ + if (authConfig?.type === "basic_token_verification") { + return res.status(200).json([]); + } + + const docs = await loadWithUrl(xFernHost); + + if (docs == null) { + // return notFoundResponse(); + return res.status(404).json([]); + } + + const root = FernNavigation.utils.convertLoadDocsForUrlResponse(docs); + const nodes: (FernNavigation.NavigationNodeWithMarkdown | FernNavigation.NavigationNodeApiLeaf)[] = []; + + FernNavigation.utils.traverseNavigation(root, (n) => { + if (FernNavigation.hasMarkdown(n) || FernNavigation.isApiLeaf(n)) { + nodes.push(n); + } + }); + + const errors: ValidationError[] = []; + + const apis: Map = new Map(); + + Object.entries(docs.definition.apis).forEach(([key, value]) => { + apis.set(key, FernNavigation.ApiDefinitionHolder.create(value)); + }); + + const featureFlags = await getFeatureFlags(xFernHost); + setMdxBundler(await getMdxBundler(featureFlags.useMdxBundler ? "mdx-bundler" : "next-mdx-remote")); + + const collectError = async (node: FernNavigation.NavigationNodeWithMetadata, markdown: string | undefined) => { + if (!markdown) { + return; + } + try { + await unsafeSerializeMdx(markdown); + } catch (e) { + errors.push({ + markdown, + message: String(e), + slug: node.slug, + }); + } + }; + + for (const node of nodes) { + const pageId = FernNavigation.utils.getPageId(node); + if (pageId != null) { + const markdown = docs.definition.pages[pageId]?.markdown; + await collectError(node, markdown); + } + + if (FernNavigation.isApiLeaf(node)) { + const api = apis.get(node.apiDefinitionId); + + if (api != null) { + if (node.type === "endpoint") { + const endpoint = api.endpoints.get(node.endpointId); + await collectError(node, endpoint?.description); + await collectError(node, endpoint?.request?.description); + await collectError(node, endpoint?.response?.description); + for (const error of endpoint?.errorsV2 ?? []) { + await collectError(node, error.description); + } + for (const example of endpoint?.examples ?? []) { + await collectError(node, example.description); + } + for (const queryParameter of endpoint?.queryParameters ?? []) { + await collectError(node, queryParameter.description); + } + for (const header of endpoint?.headers ?? []) { + await collectError(node, header.description); + } + for (const pathParameter of endpoint?.path.pathParameters ?? []) { + await collectError(node, pathParameter.description); + } + } else if (node.type === "webSocket") { + const webSocket = api.webSockets.get(node.webSocketId); + await collectError(node, webSocket?.description); + for (const event of webSocket?.messages ?? []) { + await collectError(node, event.description); + } + for (const example of webSocket?.examples ?? []) { + await collectError(node, example.description); + } + for (const queryParameter of webSocket?.queryParameters ?? []) { + await collectError(node, queryParameter.description); + } + for (const header of webSocket?.headers ?? []) { + await collectError(node, header.description); + } + for (const pathParameter of webSocket?.path.pathParameters ?? []) { + await collectError(node, pathParameter.description); + } + } else if (node.type === "webhook") { + const webhook = api.webhooks.get(node.webhookId); + await collectError(node, webhook?.description); + await collectError(node, webhook?.payload?.description); + for (const header of webhook?.headers ?? []) { + await collectError(node, header.description); + } + } + } + } + } + + return res.status(200).json(errors); + } catch (err) { + // eslint-disable-next-line no-console + console.error(err); + return res.status(500).json([]); + } +}; + +export default handler;