From d9d7386f901b71fb94f876dd5c5fcbe12a01ba00 Mon Sep 17 00:00:00 2001 From: Rowan Cockett Date: Mon, 20 Jan 2025 15:25:08 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=84=20Fetch=20with=20Retry=20for=20HTM?= =?UTF-8?q?L=20build=20(#1793)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limits outgoing connections to five. See #1775, #1336 --- .changeset/late-phones-jam.md | 5 +++ packages/myst-cli/src/build/html/index.ts | 44 ++++++++++-------- packages/myst-cli/src/utils/fetchWithRetry.ts | 45 +++++++++++++++++++ packages/myst-cli/src/utils/index.ts | 1 + 4 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 .changeset/late-phones-jam.md create mode 100644 packages/myst-cli/src/utils/fetchWithRetry.ts diff --git a/.changeset/late-phones-jam.md b/.changeset/late-phones-jam.md new file mode 100644 index 000000000..a3340e5ca --- /dev/null +++ b/.changeset/late-phones-jam.md @@ -0,0 +1,5 @@ +--- +"myst-cli": patch +--- + +Retry html pages build and limit initial outgoing connections. diff --git a/packages/myst-cli/src/build/html/index.ts b/packages/myst-cli/src/build/html/index.ts index b2df23021..9daece843 100644 --- a/packages/myst-cli/src/build/html/index.ts +++ b/packages/myst-cli/src/build/html/index.ts @@ -9,6 +9,10 @@ import type { StartOptions } from '../site/start.js'; import { startServer } from '../site/start.js'; import { getSiteTemplate } from '../site/template.js'; import { slugToUrl } from 'myst-common'; +import pLimit from 'p-limit'; +import { fetchWithRetry } from '../../utils/fetchWithRetry.js'; + +const limitConnections = pLimit(5); export async function currentSiteRoutes( session: ISession, @@ -141,25 +145,27 @@ export async function buildHtml(session: ISession, opts: StartOptions) { // Fetch all HTML pages and assets by the template await Promise.all( - routes.map(async (route) => { - const resp = await session.fetch(route.url); - if (!resp.ok) { - session.log.error(`Error fetching ${route.url}`); - return; - } - if (route.binary && resp.body) { - await new Promise((resolve) => { - const filename = path.join(htmlDir, route.path); - if (!fs.existsSync(filename)) fs.mkdirSync(path.dirname(filename), { recursive: true }); - const fileWriteStream = fs.createWriteStream(filename); - resp.body!.pipe(fileWriteStream); - fileWriteStream.on('finish', resolve); - }); - } else { - const content = await resp.text(); - writeFileToFolder(path.join(htmlDir, route.path), content); - } - }), + routes.map(async (route) => + limitConnections(async () => { + const resp = await fetchWithRetry(session, route.url); + if (!resp.ok) { + session.log.error(`Error fetching ${route.url}`); + return; + } + if (route.binary && resp.body) { + await new Promise((resolve) => { + const filename = path.join(htmlDir, route.path); + if (!fs.existsSync(filename)) fs.mkdirSync(path.dirname(filename), { recursive: true }); + const fileWriteStream = fs.createWriteStream(filename); + resp.body!.pipe(fileWriteStream); + fileWriteStream.on('finish', resolve); + }); + } else { + const content = await resp.text(); + writeFileToFolder(path.join(htmlDir, route.path), content); + } + }), + ), ); appServer.stop(); diff --git a/packages/myst-cli/src/utils/fetchWithRetry.ts b/packages/myst-cli/src/utils/fetchWithRetry.ts new file mode 100644 index 000000000..52c501824 --- /dev/null +++ b/packages/myst-cli/src/utils/fetchWithRetry.ts @@ -0,0 +1,45 @@ +import type { RequestInfo, RequestInit, Response } from 'node-fetch'; +import type { ISession } from '../session/types.js'; + +/** + * Recursively fetch a URL with retry and exponential backoff. + */ +export async function fetchWithRetry( + session: Pick, + /** The URL to fetch. */ + url: URL | RequestInfo, + /** Options to pass to fetch (e.g., headers, method). */ + options?: RequestInit, + /** How many times total to attempt the fetch. */ + maxRetries = 3, + /** The current attempt number. */ + attempt = 1, + /** The current backoff duration in milliseconds. */ + backoff = 250, +): Promise { + try { + const resp = await session.fetch(url, options); + if (resp.ok) { + // If it's a 2xx response, we consider it a success and return it + return resp; + } else { + // For non-2xx, we treat it as a failure that triggers a retry + session.log.warn( + `Fetch of ${url} failed with HTTP status ${resp.status} for URL: ${url} (Attempt #${attempt})`, + ); + } + } catch (error) { + // This covers network failures and other errors that cause fetch to reject + session.log.warn(`Fetch of ${url} threw an error (Attempt #${attempt})`, error); + } + + // If we haven't reached the max retries, wait and recurse + if (attempt < maxRetries) { + session.log.debug(`Waiting ${backoff}ms before retry #${attempt + 1}...`); + await new Promise((resolve) => setTimeout(resolve, backoff)); + return fetchWithRetry(session, url, options, maxRetries, attempt + 1, backoff * 2); + } + + // If we made it here, all retries have been exhausted + throw new Error(`Failed to fetch ${url} after ${maxRetries} attempts.`); +} diff --git a/packages/myst-cli/src/utils/index.ts b/packages/myst-cli/src/utils/index.ts index 009648f00..c9d2b5fdc 100644 --- a/packages/myst-cli/src/utils/index.ts +++ b/packages/myst-cli/src/utils/index.ts @@ -16,6 +16,7 @@ export * from './toc.js'; export * from './uniqueArray.js'; export * from './github.js'; export * from './whiteLabelling.js'; +export * from './fetchWithRetry.js'; export * as ffmpeg from './ffmpeg.js'; export * as imagemagick from './imagemagick.js';