Skip to content

Commit

Permalink
feat: if a file is greater than 1mb, upload to S3 (#580)
Browse files Browse the repository at this point in the history
  • Loading branch information
@abvthecity
abvthecity authored Mar 26, 2024
1 parent 94fff24 commit f3a844c
Show file tree
Hide file tree
Showing 11 changed files with 204 additions and 66 deletions.
4 changes: 2 additions & 2 deletions packages/commons/core-utils/src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
export { visitObject, type ObjectPropertiesVisitor } from "./ObjectPropertiesVisitor";
export { addPrefixToString } from "./addPrefixToString";
export { assertNever, assertNeverNoThrow } from "./assertNever";
export { assertVoidNoThrow } from "./assertVoidNoThrow";
export { delay } from "./delay/delay";
export { withMinimumTime } from "./delay/withMinimumTime";
export { EMPTY_ARRAY, EMPTY_OBJECT } from "./empty";
export { identity } from "./identity";
export { isNonNullish } from "./isNonNullish";
export { assertNonNullish, isNonNullish } from "./isNonNullish";
export { noop } from "./noop";
export { visitObject, type ObjectPropertiesVisitor } from "./ObjectPropertiesVisitor";
export { entries, type Entries } from "./objects/entries";
export { isPlainObject } from "./objects/isPlainObject";
export { keys } from "./objects/keys";
Expand Down
6 changes: 6 additions & 0 deletions packages/commons/core-utils/src/isNonNullish.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
export function isNonNullish<T>(x: T | null | undefined): x is T {
return x != null;
}

export function assertNonNullish<T>(x: T | null | undefined, message?: string): asserts x is T {
if (x == null) {
throw new Error(message ?? "Value is null or undefined");
}
}
36 changes: 13 additions & 23 deletions packages/ui/app/src/api-playground/PlaygroundContext.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ import { useAtom } from "jotai";
import { atomWithStorage } from "jotai/utils";
import { mapValues, noop } from "lodash-es";
import dynamic from "next/dynamic";
import { FC, PropsWithChildren, createContext, useCallback, useContext, useEffect, useMemo, useState } from "react";
import { FC, PropsWithChildren, createContext, useCallback, useContext, useMemo, useState } from "react";
import { resolve } from "url";
import { capturePosthogEvent } from "../analytics/posthog";
import { FeatureFlags, useFeatureFlags } from "../contexts/FeatureFlagContext";
import { useFeatureFlags } from "../contexts/FeatureFlagContext";
import { useDocsContext } from "../contexts/docs-context/useDocsContext";
import { useNavigationContext } from "../contexts/navigation-context";
import { APIS } from "../sidebar/atom";
import {
Expand Down Expand Up @@ -51,24 +53,10 @@ export const PLAYGROUND_FORM_STATE_ATOM = atomWithStorage<Record<string, Playgro
export const PlaygroundContextProvider: FC<PropsWithChildren> = ({ children }) => {
const { isApiPlaygroundEnabled } = useFeatureFlags();
const [apis, setApis] = useAtom(APIS);
const { domain, basePath, selectedSlug } = useNavigationContext();
const { basePath } = useDocsContext();
const { selectedSlug } = useNavigationContext();
const [selectionState, setSelectionState] = useState<PlaygroundSelectionState | undefined>();

const [isPlaygroundEnabled, setIsPlaygroundEnabled] = useState<boolean>(isApiPlaygroundEnabled);

useEffect(() => {
if (!isPlaygroundEnabled) {
fetch(`${basePath != null ? `https://${domain}` : ""}/api/fern-docs/feature-flags`, {
headers: { "x-fern-host": domain },
mode: "no-cors",
})
.then((r) => r.json())
.then((data: FeatureFlags) => setIsPlaygroundEnabled(data.isApiPlaygroundEnabled))
// eslint-disable-next-line no-console
.catch(console.error);
}
}, [basePath, domain, isPlaygroundEnabled]);

const flattenedApis = useMemo(() => mapValues(apis, flattenRootPackage), [apis]);

const [isPlaygroundOpen, setPlaygroundOpen] = useAtom(PLAYGROUND_OPEN_ATOM);
Expand All @@ -78,7 +66,7 @@ export const PlaygroundContextProvider: FC<PropsWithChildren> = ({ children }) =
const expandPlayground = useCallback(() => {
capturePosthogEvent("api_playground_opened");
setPlaygroundHeight((currentHeight) => {
const halfWindowHeight = window.innerHeight / 2;
const halfWindowHeight: number = window.innerHeight / 2;
return currentHeight < halfWindowHeight ? halfWindowHeight : currentHeight;
});
setPlaygroundOpen(true);
Expand All @@ -90,8 +78,10 @@ export const PlaygroundContextProvider: FC<PropsWithChildren> = ({ children }) =
let matchedPackage = flattenedApis[newSelectionState.api];
if (matchedPackage == null) {
const r = await fetch(
"/api/fern-docs/resolve-api?path=/" + selectedSlug + "&api=" + newSelectionState.api,
{ mode: "no-cors" },
resolve(
basePath ?? "",
"/api/fern-docs/resolve-api?path=/" + selectedSlug + "&api=" + newSelectionState.api,
),
);

const data: ResolvedRootPackage | null = await r.json();
Expand Down Expand Up @@ -141,10 +131,10 @@ export const PlaygroundContextProvider: FC<PropsWithChildren> = ({ children }) =
});
}
},
[expandPlayground, flattenedApis, globalFormState, selectedSlug, setApis, setGlobalFormState],
[basePath, expandPlayground, flattenedApis, globalFormState, selectedSlug, setApis, setGlobalFormState],
);

if (!isPlaygroundEnabled) {
if (!isApiPlaygroundEnabled) {
return <>{children}</>;
}

Expand Down
41 changes: 33 additions & 8 deletions packages/ui/app/src/api-playground/PlaygroundEndpoint.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ export const PlaygroundEndpoint: FC<PlaygroundEndpointProps> = ({
url: buildEndpointUrl(endpoint, formState),
method: endpoint.method,
headers: buildUnredactedHeaders(endpoint, formState),
body: await serializeFormStateBody(formState.body),
body: await serializeFormStateBody(formState.body, basePath),
};
if (endpoint.responseBody?.shape.type === "stream") {
const [res, stream] = await executeProxyStream(req, basePath);
Expand Down Expand Up @@ -277,6 +277,7 @@ export const PlaygroundEndpoint: FC<PlaygroundEndpointProps> = ({

async function serializeFormStateBody(
body: PlaygroundFormStateBody | undefined,
basePath: string | undefined,
): Promise<ProxyRequest.SerializableBody | undefined> {
if (body == null) {
return undefined;
Expand All @@ -292,13 +293,15 @@ async function serializeFormStateBody(
case "file":
formDataValue[key] = {
type: "file",
value: await serializeFile(value.value),
value: await serializeFile(value.value, basePath),
};
break;
case "fileArray":
formDataValue[key] = {
type: "fileArray",
value: (await Promise.all(value.value.map(serializeFile))).filter(isNonNullish),
value: (
await Promise.all(value.value.map((value) => serializeFile(value, basePath)))
).filter(isNonNullish),
};
break;
case "json":
Expand All @@ -311,22 +314,44 @@ async function serializeFormStateBody(
return { type: "form-data", value: formDataValue };
}
case "octet-stream":
return { type: "octet-stream", value: await serializeFile(body.value) };
return { type: "octet-stream", value: await serializeFile(body.value, basePath) };
default:
assertNever(body);
}
}

function blobToDataURL(blob: Blob) {
async function blobToDataURL(file: File, basePath: string = "") {
// vercel edge function has a maximum request size of 4.5MB, so we need to upload large files to S3
// if blob is larger than 1MB, we will upload it to S3 and return the URL
// TODO: we should probably measure that the _entire_ request is less than 4.5MB
if (file.size > 1024 * 1024) {
const response = await fetch(resolve(basePath, `/api/fern-docs/upload?file=${encodeURIComponent(file.name)}`), {
method: "GET",
});

const { put, get } = (await response.json()) as { put: string; get: string };

await fetch(put, {
method: "PUT",
body: file,
headers: { "Content-Type": file.type },
});

return get;
}

return new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onloadend = () => resolve(reader.result as string);
reader.onerror = reject;
reader.readAsDataURL(blob);
reader.readAsDataURL(file);
});
}

async function serializeFile(file: File | undefined): Promise<SerializableFile | undefined> {
async function serializeFile(
file: File | undefined,
basePath: string | undefined,
): Promise<SerializableFile | undefined> {
if (file == null) {
return undefined;
}
Expand All @@ -335,6 +360,6 @@ async function serializeFile(file: File | undefined): Promise<SerializableFile |
lastModified: file.lastModified,
size: file.size,
type: file.type,
dataUrl: await blobToDataURL(file),
dataUrl: await blobToDataURL(file, basePath),
};
}
8 changes: 5 additions & 3 deletions packages/ui/docs-bundle/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
"lint:fern-prod": "pnpm turbo build && pnpm env:fern-prod next lint"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.335.0",
"@aws-sdk/s3-request-presigner": "^3.335.0",
"@fern-api/fdr-sdk": "workspace:*",
"@fern-api/venus-api-sdk": "^0.1.0",
"@fern-ui/core-utils": "workspace:*",
Expand All @@ -59,21 +61,21 @@
"@fern-platform/configs": "workspace:*",
"@next/bundle-analyzer": "^14.0.3",
"@tailwindcss/typography": "^0.5.10",
"@types/node-fetch": "2.6.9",
"@types/node": "^18.7.18",
"@types/node-fetch": "2.6.9",
"@types/react": "^18.0.20",
"@types/react-dom": "^18.2.18",
"autoprefixer": "^10.4.16",
"depcheck": "^1.4.3",
"env-cmd": "toddbluhm/env-cmd",
"eslint": "^8.56.0",
"vitest": "^1.4.0",
"organize-imports-cli": "^0.10.0",
"postcss": "^8.4.33",
"prettier": "^3.2.4",
"sass": "^1.70.0",
"stylelint": "^16.1.0",
"tailwindcss": "^3.4.1",
"typescript": "5.4.3"
"typescript": "5.4.3",
"vitest": "^1.4.0"
}
}
72 changes: 42 additions & 30 deletions packages/ui/docs-bundle/src/pages/api/fern-docs/proxy/rest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,35 +39,46 @@ async function buildRequestBody(body: ProxyRequest.SerializableBody | undefined)
case "json":
return JSON.stringify(body.value);
case "form-data": {
const formData = new FormData();
for (const [key, value] of Object.entries(body.value)) {
switch (value.type) {
case "file":
if (value.value != null) {
const base64 = value.value.dataUrl;
const blob = await dataURLtoBlob(base64);
const file = new File([blob], value.value.name, { type: value.value.type });
formData.append(key, file);
}
break;
case "fileArray": {
const files = await Promise.all(
value.value.map(async (serializedFile) => {
const base64 = serializedFile.dataUrl;
const formEntries = await Promise.all(
Object.entries(body.value).map(async ([key, value]) => {
switch (value.type) {
case "file":
if (value.value != null) {
const base64 = value.value.dataUrl;
const blob = await dataURLtoBlob(base64);
return new File([blob], serializedFile.name, { type: serializedFile.type });
}),
);
files.forEach((file) => formData.append(key, file, file.name));
break;
const file = new File([blob], value.value.name, { type: value.value.type });
return [key, file] as const;
}
return [key, null] as const;
case "fileArray": {
const files = await Promise.all(
value.value.map(async (serializedFile) => {
const base64 = serializedFile.dataUrl;
const blob = await dataURLtoBlob(base64);
return new File([blob], serializedFile.name, { type: serializedFile.type });
}),
);
return [key, files] as const;
}
case "json":
return [key, JSON.stringify(value.value)] as const;
default:
assertNever(value);
}
case "json":
formData.append(key, JSON.stringify(value.value));
break;
default:
assertNever(value);
}),
);

const formData = new FormData();
formEntries.forEach(([key, value]) => {
if (value == null) {
return;
}
}
if (Array.isArray(value)) {
value.forEach((file) => formData.append(key, file, file.name));
} else {
formData.append(key, value);
}
});
return formData;
}
case "octet-stream": {
Expand All @@ -87,24 +98,26 @@ export default async function POST(req: NextRequest): Promise<NextResponse> {
if (req.method !== "POST") {
return new NextResponse(null, { status: 405 });
}
const startTime = Date.now();

// eslint-disable-next-line no-console
console.log("Starting proxy request to", req.url);

try {
const proxyRequest = (await req.json()) as ProxyRequest;
const requestBody = await buildRequestBody(proxyRequest.body);
const headers = new Headers(proxyRequest.headers);

// omit content-type for multipart/form-data so that fetch can set it automatically with the boundary
if (headers.get("Content-Type")?.toLowerCase().includes("multipart/form-data")) {
headers.delete("Content-Type");
}

const startTime = Date.now();

const response = await fetch(proxyRequest.url, {
method: proxyRequest.method,
headers,
body: await buildRequestBody(proxyRequest.body),
body: requestBody,
});

// eslint-disable-next-line no-console
Expand Down Expand Up @@ -147,12 +160,11 @@ export default async function POST(req: NextRequest): Promise<NextResponse> {
} catch (err) {
// eslint-disable-next-line no-console
console.error(err);
const endTime = Date.now();

return jsonResponse<ProxyResponse>(500, {
error: true,
status: 500,
time: endTime - startTime,
time: -1,
size: null,
});
}
Expand Down
36 changes: 36 additions & 0 deletions packages/ui/docs-bundle/src/pages/api/fern-docs/upload.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { NextRequest, NextResponse } from "next/server";
import { createGetSignedUrl, createPutSignedUrl } from "../../../utils/createSignedUrl";

export const runtime = "edge";
export const maxDuration = 5;

export default async function GET(req: NextRequest): Promise<NextResponse> {
if (req.method !== "GET") {
return new NextResponse(null, { status: 405 });
}

const domain = req.nextUrl.hostname;
const time: string = new Date().toISOString();
const file = req.nextUrl.searchParams.get("file");

if (file == null) {
return new NextResponse(null, { status: 400 });
}

try {
const key = constructS3Key(domain, time, file);
const [put, get] = await Promise.all([
await createPutSignedUrl(key, 60), // 1 minute
await createGetSignedUrl(key, 60 * 5), // 5 minutes
]);
return NextResponse.json({ put, get }, { headers: { "Cache-Control": "public, max-age=60" } });
} catch (err) {
// eslint-disable-next-line no-console
console.error("Failed to create signed URL", err);
return new NextResponse(null, { status: 500 });
}
}

function constructS3Key(domain: string, time: string, file: string): string {
return `${domain}/user-upload/${time}/${file}`;
}
Loading

0 comments on commit f3a844c

Please sign in to comment.