From 747181f169adeedbb81e48acfcb1f2fa140eadf3 Mon Sep 17 00:00:00 2001 From: dsinghvi Date: Fri, 11 Oct 2024 00:18:18 -0400 Subject: [PATCH] fix: make sure endpoints are cached in nextjs route on register --- .../docs/v2/getDocsWriteV2Service.ts | 82 ++++++++++++------- .../algolia/AlgoliaSearchRecordGenerator.ts | 6 +- .../src/services/algolia/AlgoliaService.ts | 4 +- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts b/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts index fe5e95cc37..1f13d1da8b 100644 --- a/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts +++ b/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts @@ -1,4 +1,5 @@ import { APIV1Db, convertDocsDefinitionToDb, DocsV1Db, DocsV1Write, FdrAPI } from "@fern-api/fdr-sdk"; +import { isNonNullish } from "@fern-api/ui-core-utils"; import { AuthType } from "@prisma/client"; import urlJoin from "url-join"; import { v4 as uuidv4 } from "uuid"; @@ -147,14 +148,24 @@ export function getDocsWriteV2Service(app: FdrApplication): DocsV2WriteService { files: docsRegistrationInfo.s3FileInfos, }); - const apiDefinitionsById = await (async () => { - const apiIdDefinitionTuples = await Promise.all( - dbDocsDefinition.referencedApis.map( - async (id) => [id, await app.services.db.getApiDefinition(id)] as const, - ), - ); - return new Map(apiIdDefinitionTuples) as Map; - })(); + const apiDefinitions = ( + await Promise.all( + dbDocsDefinition.referencedApis.map(async (id) => await app.services.db.getApiDefinition(id)), + ) + ).filter(isNonNullish); + const apiDefinitionsById = Object.fromEntries( + apiDefinitions.map((definition) => [definition.id, definition]), + ); + + const warmEndpointCachePromises = apiDefinitions.flatMap((apiDefinition) => { + return Object.entries(apiDefinition.subpackages).flatMap(([id, subpackage]) => { + return subpackage.endpoints.map(async (endpoint) => { + return await fetch( + `${docsRegistrationInfo.fernUrl.getFullUrl()}/api/fern-docs/api-definition/${apiDefinition.id}/endpoint/${endpoint.id}`, + ); + }); + }); + }); const indexSegments = await uploadToAlgoliaForRegistration( app, @@ -175,19 +186,30 @@ export function getDocsWriteV2Service(app: FdrApplication): DocsV2WriteService { */ const urls = [docsRegistrationInfo.fernUrl, ...docsRegistrationInfo.customUrls]; - await Promise.all( - urls.map(async (baseUrl) => { - const results = await app.services.revalidator.revalidate({ baseUrl, app }); - if (results.failed.length === 0 && !results.revalidationFailed) { - app.logger.info(`Successfully revalidated ${results.successful.length} paths.`); - } else { - await app.services.slack.notifyFailedToRevalidatePaths({ - domain: baseUrl.getFullUrl(), - paths: results, - }); - } - }), - ); + try { + await Promise.all( + urls.map(async (baseUrl) => { + const results = await app.services.revalidator.revalidate({ baseUrl, app }); + if (results.failed.length === 0 && !results.revalidationFailed) { + app.logger.info(`Successfully revalidated ${results.successful.length} paths.`); + } else { + await app.services.slack.notifyFailedToRevalidatePaths({ + domain: baseUrl.getFullUrl(), + paths: results, + }); + } + }), + ); + } catch (e) { + app.logger.error(`Error while trying to revalidate docs for ${docsRegistrationInfo.fernUrl}`, e); + await app.services.slack.notifyFailedToRegisterDocs({ + domain: docsRegistrationInfo.fernUrl.getFullUrl(), + err: e, + }); + throw e; + } + + await Promise.all(warmEndpointCachePromises); return res.send(); } catch (e) { @@ -212,14 +234,16 @@ export function getDocsWriteV2Service(app: FdrApplication): DocsV2WriteService { throw new ReindexNotAllowedError(); } - const apiDefinitionsById = await (async () => { - const apiIdDefinitionTuples = await Promise.all( + const apiDefinitions = ( + await Promise.all( response.docsDefinition.referencedApis.map( - async (id) => [id, await app.services.db.getApiDefinition(id)] as const, + async (id) => await app.services.db.getApiDefinition(id), ), - ); - return new Map(apiIdDefinitionTuples) as Map; - })(); + ) + ).filter(isNonNullish); + const apiDefinitionsById = Object.fromEntries( + apiDefinitions.map((definition) => [definition.id, definition]), + ); // step 2. create new index segments in algolia const indexSegments = await uploadToAlgolia( @@ -245,7 +269,7 @@ async function uploadToAlgoliaForRegistration( app: FdrApplication, docsRegistrationInfo: DocsRegistrationInfo, dbDocsDefinition: WithoutQuestionMarks, - apiDefinitionsById: Map, + apiDefinitionsById: Record, ): Promise { // TODO: make sure to store private docs index into user-restricted algolia index // see https://www.algolia.com/doc/guides/security/api-keys/how-to/user-restricted-access-to-data/ @@ -265,7 +289,7 @@ async function uploadToAlgolia( app: FdrApplication, url: ParsedBaseUrl, dbDocsDefinition: WithoutQuestionMarks, - apiDefinitionsById: Map, + apiDefinitionsById: Record, ): Promise { app.logger.debug(`[${url.getFullUrl()}] Generating new index segments`); const generateNewIndexSegmentsResult = app.services.algoliaIndexSegmentManager.generateIndexSegmentsForDefinition({ diff --git a/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts b/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts index 09e7dbd1c0..b09f8adaa0 100644 --- a/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts +++ b/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts @@ -22,7 +22,7 @@ import type { AlgoliaSearchRecord, IndexSegment } from "./types"; interface AlgoliaSearchRecordGeneratorConfig { docsDefinition: DocsV1Db.DocsDefinitionDb; - apiDefinitionsById: Map; + apiDefinitionsById: Record; } export class AlgoliaSearchRecordGenerator { @@ -140,7 +140,7 @@ export class AlgoliaSearchRecordGenerator { const records: AlgoliaSearchRecord[] = []; const api = item; const apiId = api.api; - const apiDef = this.config.apiDefinitionsById.get(apiId); + const apiDef = this.config.apiDefinitionsById[apiId]; if (apiDef != null) { records.push( ...this.generateAlgoliaSearchRecordsForApiDefinition( @@ -232,7 +232,7 @@ export class AlgoliaSearchRecordGenerator { root: FernNavigation.V1.ApiReferenceNode, context: NavigationContext, ): AlgoliaSearchRecord[] { - const api = this.config.apiDefinitionsById.get(root.apiDefinitionId); + const api = this.config.apiDefinitionsById[root.apiDefinitionId]; if (api == null) { LOGGER.error("Failed to find API definition for API reference node. id=", root.apiDefinitionId); } diff --git a/servers/fdr/src/services/algolia/AlgoliaService.ts b/servers/fdr/src/services/algolia/AlgoliaService.ts index 974d8efd22..2c82c9346d 100644 --- a/servers/fdr/src/services/algolia/AlgoliaService.ts +++ b/servers/fdr/src/services/algolia/AlgoliaService.ts @@ -11,7 +11,7 @@ export interface AlgoliaService { generateSearchRecords(params: { url: string; docsDefinition: DocsV1Db.DocsDefinitionDb; - apiDefinitionsById: Map; + apiDefinitionsById: Record; configSegmentTuples: ConfigSegmentTuple[]; }): Promise; @@ -48,7 +48,7 @@ export class AlgoliaServiceImpl implements AlgoliaService { }: { url: string; docsDefinition: DocsV1Db.DocsDefinitionDb; - apiDefinitionsById: Map; + apiDefinitionsById: Record; configSegmentTuples: ConfigSegmentTuple[]; }) { return configSegmentTuples.flatMap(([config, indexSegment]) => {