From f3a844c677dd7a0299c9bbbed92c7a6b5941e142 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Wed, 27 Mar 2024 07:54:38 +1100 Subject: [PATCH] feat: if a file is greater than 1mb, upload to S3 (#580) --- packages/commons/core-utils/src/index.ts | 4 +- .../commons/core-utils/src/isNonNullish.ts | 6 ++ .../src/api-playground/PlaygroundContext.tsx | 36 ++++------ .../src/api-playground/PlaygroundEndpoint.tsx | 41 ++++++++--- packages/ui/docs-bundle/package.json | 8 ++- .../src/pages/api/fern-docs/proxy/rest.ts | 72 +++++++++++-------- .../src/pages/api/fern-docs/upload.ts | 36 ++++++++++ .../docs-bundle/src/utils/createSignedUrl.ts | 26 +++++++ packages/ui/docs-bundle/src/utils/once.ts | 13 ++++ .../docs-bundle/src/utils/provideS3Client.ts | 22 ++++++ pnpm-lock.yaml | 6 ++ 11 files changed, 204 insertions(+), 66 deletions(-) create mode 100644 packages/ui/docs-bundle/src/pages/api/fern-docs/upload.ts create mode 100644 packages/ui/docs-bundle/src/utils/createSignedUrl.ts create mode 100644 packages/ui/docs-bundle/src/utils/once.ts create mode 100644 packages/ui/docs-bundle/src/utils/provideS3Client.ts diff --git a/packages/commons/core-utils/src/index.ts b/packages/commons/core-utils/src/index.ts index 31ec12ee93..78a328929d 100644 --- a/packages/commons/core-utils/src/index.ts +++ b/packages/commons/core-utils/src/index.ts @@ -1,3 +1,4 @@ +export { visitObject, type ObjectPropertiesVisitor } from "./ObjectPropertiesVisitor"; export { addPrefixToString } from "./addPrefixToString"; export { assertNever, assertNeverNoThrow } from "./assertNever"; export { assertVoidNoThrow } from "./assertVoidNoThrow"; @@ -5,9 +6,8 @@ export { delay } from "./delay/delay"; export { withMinimumTime } from "./delay/withMinimumTime"; export { EMPTY_ARRAY, EMPTY_OBJECT } from "./empty"; export { identity } from "./identity"; -export { isNonNullish } from "./isNonNullish"; +export { assertNonNullish, isNonNullish } from "./isNonNullish"; export { noop } from "./noop"; -export { visitObject, type ObjectPropertiesVisitor } from "./ObjectPropertiesVisitor"; export { entries, type Entries } from "./objects/entries"; export { isPlainObject } from "./objects/isPlainObject"; export { keys } from "./objects/keys"; diff --git a/packages/commons/core-utils/src/isNonNullish.ts b/packages/commons/core-utils/src/isNonNullish.ts index a13ecf5769..adbf6c0755 100644 --- a/packages/commons/core-utils/src/isNonNullish.ts +++ b/packages/commons/core-utils/src/isNonNullish.ts @@ -1,3 +1,9 @@ export function isNonNullish(x: T | null | undefined): x is T { return x != null; } + +export function assertNonNullish(x: T | null | undefined, message?: string): asserts x is T { + if (x == null) { + throw new Error(message ?? "Value is null or undefined"); + } +} diff --git a/packages/ui/app/src/api-playground/PlaygroundContext.tsx b/packages/ui/app/src/api-playground/PlaygroundContext.tsx index b3cb666dd4..5dcc6181be 100644 --- a/packages/ui/app/src/api-playground/PlaygroundContext.tsx +++ b/packages/ui/app/src/api-playground/PlaygroundContext.tsx @@ -2,9 +2,11 @@ import { useAtom } from "jotai"; import { atomWithStorage } from "jotai/utils"; import { mapValues, noop } from "lodash-es"; import dynamic from "next/dynamic"; -import { FC, PropsWithChildren, createContext, useCallback, useContext, useEffect, useMemo, useState } from "react"; +import { FC, PropsWithChildren, createContext, useCallback, useContext, useMemo, useState } from "react"; +import { resolve } from "url"; import { capturePosthogEvent } from "../analytics/posthog"; -import { FeatureFlags, useFeatureFlags } from "../contexts/FeatureFlagContext"; +import { useFeatureFlags } from "../contexts/FeatureFlagContext"; +import { useDocsContext } from "../contexts/docs-context/useDocsContext"; import { useNavigationContext } from "../contexts/navigation-context"; import { APIS } from "../sidebar/atom"; import { @@ -51,24 +53,10 @@ export const PLAYGROUND_FORM_STATE_ATOM = atomWithStorage = ({ children }) => { const { isApiPlaygroundEnabled } = useFeatureFlags(); const [apis, setApis] = useAtom(APIS); - const { domain, basePath, selectedSlug } = useNavigationContext(); + const { basePath } = useDocsContext(); + const { selectedSlug } = useNavigationContext(); const [selectionState, setSelectionState] = useState(); - const [isPlaygroundEnabled, setIsPlaygroundEnabled] = useState(isApiPlaygroundEnabled); - - useEffect(() => { - if (!isPlaygroundEnabled) { - fetch(`${basePath != null ? `https://${domain}` : ""}/api/fern-docs/feature-flags`, { - headers: { "x-fern-host": domain }, - mode: "no-cors", - }) - .then((r) => r.json()) - .then((data: FeatureFlags) => setIsPlaygroundEnabled(data.isApiPlaygroundEnabled)) - // eslint-disable-next-line no-console - .catch(console.error); - } - }, [basePath, domain, isPlaygroundEnabled]); - const flattenedApis = useMemo(() => mapValues(apis, flattenRootPackage), [apis]); const [isPlaygroundOpen, setPlaygroundOpen] = useAtom(PLAYGROUND_OPEN_ATOM); @@ -78,7 +66,7 @@ export const PlaygroundContextProvider: FC = ({ children }) = const expandPlayground = useCallback(() => { capturePosthogEvent("api_playground_opened"); setPlaygroundHeight((currentHeight) => { - const halfWindowHeight = window.innerHeight / 2; + const halfWindowHeight: number = window.innerHeight / 2; return currentHeight < halfWindowHeight ? halfWindowHeight : currentHeight; }); setPlaygroundOpen(true); @@ -90,8 +78,10 @@ export const PlaygroundContextProvider: FC = ({ children }) = let matchedPackage = flattenedApis[newSelectionState.api]; if (matchedPackage == null) { const r = await fetch( - "/api/fern-docs/resolve-api?path=/" + selectedSlug + "&api=" + newSelectionState.api, - { mode: "no-cors" }, + resolve( + basePath ?? "", + "/api/fern-docs/resolve-api?path=/" + selectedSlug + "&api=" + newSelectionState.api, + ), ); const data: ResolvedRootPackage | null = await r.json(); @@ -141,10 +131,10 @@ export const PlaygroundContextProvider: FC = ({ children }) = }); } }, - [expandPlayground, flattenedApis, globalFormState, selectedSlug, setApis, setGlobalFormState], + [basePath, expandPlayground, flattenedApis, globalFormState, selectedSlug, setApis, setGlobalFormState], ); - if (!isPlaygroundEnabled) { + if (!isApiPlaygroundEnabled) { return <>{children}; } diff --git a/packages/ui/app/src/api-playground/PlaygroundEndpoint.tsx b/packages/ui/app/src/api-playground/PlaygroundEndpoint.tsx index 7cc4085c3a..cb155cc410 100644 --- a/packages/ui/app/src/api-playground/PlaygroundEndpoint.tsx +++ b/packages/ui/app/src/api-playground/PlaygroundEndpoint.tsx @@ -160,7 +160,7 @@ export const PlaygroundEndpoint: FC = ({ url: buildEndpointUrl(endpoint, formState), method: endpoint.method, headers: buildUnredactedHeaders(endpoint, formState), - body: await serializeFormStateBody(formState.body), + body: await serializeFormStateBody(formState.body, basePath), }; if (endpoint.responseBody?.shape.type === "stream") { const [res, stream] = await executeProxyStream(req, basePath); @@ -277,6 +277,7 @@ export const PlaygroundEndpoint: FC = ({ async function serializeFormStateBody( body: PlaygroundFormStateBody | undefined, + basePath: string | undefined, ): Promise { if (body == null) { return undefined; @@ -292,13 +293,15 @@ async function serializeFormStateBody( case "file": formDataValue[key] = { type: "file", - value: await serializeFile(value.value), + value: await serializeFile(value.value, basePath), }; break; case "fileArray": formDataValue[key] = { type: "fileArray", - value: (await Promise.all(value.value.map(serializeFile))).filter(isNonNullish), + value: ( + await Promise.all(value.value.map((value) => serializeFile(value, basePath))) + ).filter(isNonNullish), }; break; case "json": @@ -311,22 +314,44 @@ async function serializeFormStateBody( return { type: "form-data", value: formDataValue }; } case "octet-stream": - return { type: "octet-stream", value: await serializeFile(body.value) }; + return { type: "octet-stream", value: await serializeFile(body.value, basePath) }; default: assertNever(body); } } -function blobToDataURL(blob: Blob) { +async function blobToDataURL(file: File, basePath: string = "") { + // vercel edge function has a maximum request size of 4.5MB, so we need to upload large files to S3 + // if blob is larger than 1MB, we will upload it to S3 and return the URL + // TODO: we should probably measure that the _entire_ request is less than 4.5MB + if (file.size > 1024 * 1024) { + const response = await fetch(resolve(basePath, `/api/fern-docs/upload?file=${encodeURIComponent(file.name)}`), { + method: "GET", + }); + + const { put, get } = (await response.json()) as { put: string; get: string }; + + await fetch(put, { + method: "PUT", + body: file, + headers: { "Content-Type": file.type }, + }); + + return get; + } + return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onloadend = () => resolve(reader.result as string); reader.onerror = reject; - reader.readAsDataURL(blob); + reader.readAsDataURL(file); }); } -async function serializeFile(file: File | undefined): Promise { +async function serializeFile( + file: File | undefined, + basePath: string | undefined, +): Promise { if (file == null) { return undefined; } @@ -335,6 +360,6 @@ async function serializeFile(file: File | undefined): Promise { - const base64 = serializedFile.dataUrl; + const formEntries = await Promise.all( + Object.entries(body.value).map(async ([key, value]) => { + switch (value.type) { + case "file": + if (value.value != null) { + const base64 = value.value.dataUrl; const blob = await dataURLtoBlob(base64); - return new File([blob], serializedFile.name, { type: serializedFile.type }); - }), - ); - files.forEach((file) => formData.append(key, file, file.name)); - break; + const file = new File([blob], value.value.name, { type: value.value.type }); + return [key, file] as const; + } + return [key, null] as const; + case "fileArray": { + const files = await Promise.all( + value.value.map(async (serializedFile) => { + const base64 = serializedFile.dataUrl; + const blob = await dataURLtoBlob(base64); + return new File([blob], serializedFile.name, { type: serializedFile.type }); + }), + ); + return [key, files] as const; + } + case "json": + return [key, JSON.stringify(value.value)] as const; + default: + assertNever(value); } - case "json": - formData.append(key, JSON.stringify(value.value)); - break; - default: - assertNever(value); + }), + ); + + const formData = new FormData(); + formEntries.forEach(([key, value]) => { + if (value == null) { + return; } - } + if (Array.isArray(value)) { + value.forEach((file) => formData.append(key, file, file.name)); + } else { + formData.append(key, value); + } + }); return formData; } case "octet-stream": { @@ -87,13 +98,13 @@ export default async function POST(req: NextRequest): Promise { if (req.method !== "POST") { return new NextResponse(null, { status: 405 }); } - const startTime = Date.now(); // eslint-disable-next-line no-console console.log("Starting proxy request to", req.url); try { const proxyRequest = (await req.json()) as ProxyRequest; + const requestBody = await buildRequestBody(proxyRequest.body); const headers = new Headers(proxyRequest.headers); // omit content-type for multipart/form-data so that fetch can set it automatically with the boundary @@ -101,10 +112,12 @@ export default async function POST(req: NextRequest): Promise { headers.delete("Content-Type"); } + const startTime = Date.now(); + const response = await fetch(proxyRequest.url, { method: proxyRequest.method, headers, - body: await buildRequestBody(proxyRequest.body), + body: requestBody, }); // eslint-disable-next-line no-console @@ -147,12 +160,11 @@ export default async function POST(req: NextRequest): Promise { } catch (err) { // eslint-disable-next-line no-console console.error(err); - const endTime = Date.now(); return jsonResponse(500, { error: true, status: 500, - time: endTime - startTime, + time: -1, size: null, }); } diff --git a/packages/ui/docs-bundle/src/pages/api/fern-docs/upload.ts b/packages/ui/docs-bundle/src/pages/api/fern-docs/upload.ts new file mode 100644 index 0000000000..dca69008b3 --- /dev/null +++ b/packages/ui/docs-bundle/src/pages/api/fern-docs/upload.ts @@ -0,0 +1,36 @@ +import { NextRequest, NextResponse } from "next/server"; +import { createGetSignedUrl, createPutSignedUrl } from "../../../utils/createSignedUrl"; + +export const runtime = "edge"; +export const maxDuration = 5; + +export default async function GET(req: NextRequest): Promise { + if (req.method !== "GET") { + return new NextResponse(null, { status: 405 }); + } + + const domain = req.nextUrl.hostname; + const time: string = new Date().toISOString(); + const file = req.nextUrl.searchParams.get("file"); + + if (file == null) { + return new NextResponse(null, { status: 400 }); + } + + try { + const key = constructS3Key(domain, time, file); + const [put, get] = await Promise.all([ + await createPutSignedUrl(key, 60), // 1 minute + await createGetSignedUrl(key, 60 * 5), // 5 minutes + ]); + return NextResponse.json({ put, get }, { headers: { "Cache-Control": "public, max-age=60" } }); + } catch (err) { + // eslint-disable-next-line no-console + console.error("Failed to create signed URL", err); + return new NextResponse(null, { status: 500 }); + } +} + +function constructS3Key(domain: string, time: string, file: string): string { + return `${domain}/user-upload/${time}/${file}`; +} diff --git a/packages/ui/docs-bundle/src/utils/createSignedUrl.ts b/packages/ui/docs-bundle/src/utils/createSignedUrl.ts new file mode 100644 index 0000000000..bcb442974b --- /dev/null +++ b/packages/ui/docs-bundle/src/utils/createSignedUrl.ts @@ -0,0 +1,26 @@ +import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { assertNonNullish } from "@fern-ui/core-utils"; +import { provideS3Client } from "./provideS3Client"; + +export const createPutSignedUrl = async (key: string, expiresIn: number = 60): Promise => { + const s3Client = provideS3Client(); + assertNonNullish(s3Client, "S3 client is not available"); + assertNonNullish(process.env.AWS_S3_BUCKET_NAME, "AWS_S3_BUCKET_NAME is not set"); + const command = new PutObjectCommand({ + Bucket: process.env.AWS_S3_BUCKET_NAME, + Key: key, + }); + return getSignedUrl(s3Client, command, { expiresIn }); +}; + +export const createGetSignedUrl = async (key: string, expiresIn: number = 60): Promise => { + const s3Client = provideS3Client(); + assertNonNullish(s3Client, "S3 client is not available"); + assertNonNullish(process.env.AWS_S3_BUCKET_NAME, "AWS_S3_BUCKET_NAME is not set"); + const command = new GetObjectCommand({ + Bucket: process.env.AWS_S3_BUCKET_NAME, + Key: key, + }); + return getSignedUrl(s3Client, command, { expiresIn }); +}; diff --git a/packages/ui/docs-bundle/src/utils/once.ts b/packages/ui/docs-bundle/src/utils/once.ts new file mode 100644 index 0000000000..52233e0e5f --- /dev/null +++ b/packages/ui/docs-bundle/src/utils/once.ts @@ -0,0 +1,13 @@ +export function once(fn: (...args: P[]) => R): (...args: P[]) => R { + let called = false; + let result: R; + + return function (this: unknown, ...args: P[]) { + if (!called) { + called = true; + result = fn.apply(this, args); + } + + return result; + }; +} diff --git a/packages/ui/docs-bundle/src/utils/provideS3Client.ts b/packages/ui/docs-bundle/src/utils/provideS3Client.ts new file mode 100644 index 0000000000..96e4f88287 --- /dev/null +++ b/packages/ui/docs-bundle/src/utils/provideS3Client.ts @@ -0,0 +1,22 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { assertNonNullish } from "@fern-ui/core-utils"; +import { once } from "./once"; + +export const provideS3Client = once((): S3Client | undefined => { + try { + assertNonNullish(process.env.AWS_ACCESS_KEY_ID, "AWS_ACCESS_KEY_ID is not set"); + assertNonNullish(process.env.AWS_SECRET_ACCESS_KEY, "AWS_SECRET_ACCESS_KEY is not set"); + return new S3Client({ + endpoint: process.env.AWS_S3_ENDPOINT, // this can be undefined + region: process.env.AWS_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(err); + } + return undefined; +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 29df65a9d3..3136c79369 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -931,6 +931,12 @@ importers: packages/ui/docs-bundle: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.335.0 + version: 3.537.0 + '@aws-sdk/s3-request-presigner': + specifier: ^3.335.0 + version: 3.537.0 '@fern-api/fdr-sdk': specifier: workspace:* version: link:../../fdr-sdk