perf: ⚡️ error on tgz fetch, and use fetch to get info

charlzyx · Feb 19, 2024 · fd79d46 · fd79d46
1 parent efb695d
commit fd79d46
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 71 deletions.
diff --git a/src/index.ts b/src/index.ts
@@ -82,13 +82,13 @@ app
         .use(router),
   )
 
-  .onError(({ code, error }) => {
+  .onError(({ code, error, path }) => {
     const resp = new Response(
       err("reason", error?.message || error.toString()),
     );
     resp.headers.set("Content-Type", "text/html; charset=utf8");
     console.log(
-      `ERROR: ${new Date().toLocaleDateString()}\n`,
+      `[Error]: ${new Date().toLocaleString()} ${path}\n`,
       error.stack,
       error.message,
     );

diff --git a/src/templates/BUNPKG.html b/src/templates/BUNPKG.html
@@ -7524,19 +7524,7 @@ <h2 class="vocs_H2 vocs_Heading" data-immersive-translate-walked="b4cbe8f6-4615-
               <a class="vocs_Anchor vocs_Link vocs_Link_accent_underlined vocs_ExternalLink"
                 href="https://elysiajs.com/" target="_blank" rel="noopener noreferrer"
                 data-immersive-translate-walked="b4cbe8f6-4615-4714-9132-1d4713edad92"
-                data-immersive-translate-paragraph="1">Elysia<font
-                  class="notranslate immersive-translate-target-wrapper" lang="zh-CN"
-                  data-immersive-translate-translation-element-mark="1">
-                  <font class="notranslate" data-immersive-translate-translation-element-mark="1">&nbsp;</font>
-                  <font
-                    class="notranslate immersive-translate-target-translation-theme-italic immersive-translate-target-translation-inline-wrapper-theme-italic immersive-translate-target-translation-inline-wrapper"
-                    data-immersive-translate-translation-element-mark="1">
-                    <font
-                      class="notranslate immersive-translate-target-inner immersive-translate-target-translation-theme-italic-inner"
-                      data-immersive-translate-translation-element-mark="1">
-                      爱丽舍</font>
-                  </font>
-                </font></a>
+                data-immersive-translate-paragraph="1">Elysia</a>
             </li>
             <li class="vocs_ListItem" data-immersive-translate-walked="b4cbe8f6-4615-4714-9132-1d4713edad92">
               <a class="vocs_Anchor vocs_Link vocs_Link_accent_underlined vocs_ExternalLink" href="https://bun.sh/"

diff --git a/src/utils/http.ts b/src/utils/http.ts
@@ -2,9 +2,8 @@ import { IncomingMessage } from "http";
 import https, { RequestOptions } from "https";
 import http from "http";
 import { LRUCache } from "lru-cache";
-import { BunPkgConfig } from "../config";
 import { SIZE, TTL } from "./helper";
-import { URL } from "whatwg-url";
+import { BunPkgConfig } from "../config";
 
 const agent = new https.Agent({
   keepAlive: true,
@@ -17,8 +16,7 @@ export const httpCache = new LRUCache<string, string>({
   maxSize: SIZE.Gib(1),
 });
 
-// is stream can shared ?
-const conMap = {} as Record<string, Promise<IncomingMessage> | void>;
+const conMap = {} as Record<string, Promise<Response> | void>;
 
 export const tgz = (name: string) => {
   return new Promise<IncomingMessage>((resolve, reject) => {
@@ -30,7 +28,6 @@ export const tgz = (name: string) => {
           path: `/tgz/${name}`,
         },
         (res) => {
-          console.log("🚀 ~ tgz ~ res:", name);
           return resolve(res);
         },
       )
@@ -40,33 +37,28 @@ export const tgz = (name: string) => {
       });
   });
 };
-export const get = (url: string) => {
-  const { hostname, pathname, protocol, port } = new URL(url);
-  const conKey = hostname! + pathname;
-  const maybe = conMap[conKey];
+
+export const get = (url: string, json = true) => {
+  const maybe = conMap[url];
   if (maybe) {
+    // console.log("🚀 ~ 且慢 ~ maybe:", url);
+
     return maybe;
   } else {
-    const p = new Promise<IncomingMessage>((resolve, reject) => {
-      const exec = /https/.test(protocol) ? https : http;
-      exec
-        .get(
-          {
-            agent,
-            path: pathname,
-            hostname,
-            port: port,
-            headers: {
-              Authorization: `Bearer ${BunPkgConfig.npmAuthToken}`,
-            },
-          },
-          resolve,
-        )
-        .on("error", reject);
-    }).finally(() => {
-      conMap[conKey] = null!;
-    });
-    conMap[conKey] = p;
+    const headers: Record<string, string> = {
+      "User-Agent": "bunpkg",
+    };
+    if (BunPkgConfig.npmAuthToken) {
+      headers.Authorization = `Bearer ${BunPkgConfig.npmAuthToken}`;
+    }
+    const p = fetch(url, {
+      headers,
+    })
+      .then((resp) => (json ? resp.json() : resp))
+      .finally(() => {
+        conMap[url] = null!;
+      });
+    conMap[url] = p;
     return p;
   }
 };
diff --git a/src/utils/npm.ts b/src/utils/npm.ts
@@ -30,33 +30,32 @@ const cacheYou = (cacheKey: string) => {
 const queryPackageInfo = async (packageName: string) => {
   const cacheKey = `pkg-info-${packageName}`;
   const cached = cacheYou(cacheKey);
+  // console.log(`🚀 ~ queryPackageInfo ~ cacheYou: ${cached}`, cacheKey);
   if (cached) return cached;
 
   const npmRegistryURL = BunPkgConfig.npmRegistryURL;
 
-  const name = encodePackageName(packageName);
-  const infoURL = `${npmRegistryURL}/${name}`;
+  const queryInfo = async (npmRegistryURL: string) => {
+    const name = encodePackageName(packageName);
+    const infoURL = `${npmRegistryURL}/${name}`;
 
-  console.debug("Fetching package info for %s from %s", packageName, infoURL);
+    console.debug("Fetching package info for %s from %s", packageName, infoURL);
 
-  const res = await get(infoURL);
+    const resp = await get(infoURL);
+    // console.log(`🚀 ~ queryInfo ~ resp:`, resp.status);
 
-  if (res.statusCode === 200) {
-    return promiseifyStream(res).then((value) => {
-      httpCache.set(cacheKey, value, { ttl: TTL.MINUTES(1) });
-      return JSON.parse(value);
-    });
-  }
-
-  if (res.statusCode === 400) {
-    httpCache.set(cacheKey, NOT_FOUND, { ttl: TTL.MINUTES(5) });
-  }
-
-  const reason = `Error fetching info for ${packageName} (status: ${res.statusCode}`;
-
-  console.error(reason);
+    if (resp) {
+      // console.log("🚀 ~ queryPackageInfo ~ setCacheKey", cacheKey, resp);
+      httpCache.set(cacheKey, JSON.stringify(resp), { ttl: TTL.MINUTES(1) });
+      return resp;
+    } else {
+      const reason = `Error fetching package info for ${packageName} `;
+      console.error(reason);
+      return null;
+    }
+  };
 
-  return null;
+  return await queryInfo(npmRegistryURL);
 };
 
 /**
@@ -143,21 +142,28 @@ export const queryPackageTarball = async (
     : packageName;
   const pkgConfig = await queryPackageConfigOfVersion(packageName, version);
   const npmRegistryURL = BunPkgConfig.npmRegistryURL;
+
+  const tgzName = `${tarballName}-${version}.tgz`;
+
   const tarballURL = pkgConfig?.dist
     ? pkgConfig?.dist?.tarball
     : `${npmRegistryURL}/${packageName}/-/${tarballName}-${version}.tgz`;
 
-  const tgzName = `${tarballName}-${version}.tgz`;
-
   const cacheYou = await SqliteCache.tgz.read(tgzName);
   if (cacheYou) {
     return tgz(tgzName);
   } else {
     console.debug("Fetching package for %s from %s", packageName, tarballURL);
     try {
-      const buffer = await fetch(tarballURL);
-      await SqliteCache.tgz.write(tgzName, buffer as any, {});
-      return tgz(tgzName);
+      const resp = await get(tarballURL, false);
+      if (resp.status === 200) {
+        await SqliteCache.tgz.write(tgzName, resp as any, pkgConfig.dist ?? {});
+        return tgz(tgzName);
+      } else {
+        const reason = `Error download tarball for ${packageName} (${resp.status}: ${resp.statusText})`;
+        console.error(reason);
+        throw new Error(reason);
+      }
     } catch (error) {
       console.debug(
         "Fetching error %s, for %s from %s",
@@ -188,9 +194,9 @@ export const searchPackageEntry = async (
 
   if (!entry) {
     throw new Error(
-      `Cannot find entry ${filename} in ${packageName}@${packageVersion}. tried  ${tried.join(
-        "\n",
-      )}`,
+      `Cannot find entry ${filename} in ${packageName}@${packageVersion}. ${
+        Array.isArray(tried) ? tried.join("\n") : ""
+      }`,
     );
   }
   if (entry.type === "file" && entry.path !== filename) {

diff --git a/src/utils/sqlite-cache.ts b/src/utils/sqlite-cache.ts
@@ -66,6 +66,11 @@ export const cacheFactory = ({
           },
         });
         delete writeTasks[filePath];
+
+        if (meta?.size && meta.size !== size) {
+          unlink(filePath, () => {});
+          throw new Error(`Size Error will write file ${filePath}`);
+        }
       });
       return writeTasks[filePath];
     },