Merge pull request #4511 from owid/fix-indexing-and-error-capturing

🐛 Fix algolia indexing and error capturing
owid · Jan 31, 2025 · 15838ee · 15838ee
2 parents 1401cf2 + 6e5369e
commit 15838ee
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 26 deletions.
diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts
@@ -2,6 +2,7 @@
 // set up before any errors are thrown.
 import "../../serverUtils/instrument.js"
 
+import * as Sentry from "@sentry/node"
 import * as db from "../../db/db.js"
 import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js"
 import { getAlgoliaClient } from "./configureAlgolia.js"
@@ -27,9 +28,9 @@ const indexChartsToAlgolia = async () => {
     await index.replaceAllObjects(records)
 }
 
-process.on("unhandledRejection", (e) => {
-    console.error(e)
+indexChartsToAlgolia().catch(async (e) => {
+    console.error("Error in indexChartsToAlgolia:", e)
+    Sentry.captureException(e)
+    await Sentry.close()
     process.exit(1)
 })
-
-void indexChartsToAlgolia()
diff --git a/baker/algolia/indexExplorerViewsAndChartsToAlgolia.ts b/baker/algolia/indexExplorerViewsAndChartsToAlgolia.ts
@@ -2,6 +2,7 @@
 // set up before any errors are thrown.
 import "../../serverUtils/instrument.js"
 
+import * as Sentry from "@sentry/node"
 import * as db from "../../db/db.js"
 import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js"
 import { getAlgoliaClient } from "./configureAlgolia.js"
@@ -55,9 +56,9 @@ const indexExplorerViewsAndChartsToAlgolia = async () => {
     console.log(`Indexing complete`)
 }
 
-process.on("unhandledRejection", (e) => {
-    console.error(e)
+indexExplorerViewsAndChartsToAlgolia().catch(async (e) => {
+    console.error("Error in indexExplorerViewsAndChartsToAlgolia:", e)
+    Sentry.captureException(e)
+    await Sentry.close()
     process.exit(1)
 })
-
-void indexExplorerViewsAndChartsToAlgolia()
diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts
@@ -2,6 +2,7 @@
 // set up before any errors are thrown.
 import "../../serverUtils/instrument.js"
 
+import * as Sentry from "@sentry/node"
 import * as db from "../../db/db.js"
 import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js"
 import { getAlgoliaClient } from "./configureAlgolia.js"
@@ -30,9 +31,9 @@ const indexExplorerViewsToAlgolia = async () => {
     console.log(`Indexing complete`)
 }
 
-process.on("unhandledRejection", (e) => {
-    console.error(e)
+indexExplorerViewsToAlgolia().catch(async (e) => {
+    console.error("Error in indexExplorerViewsToAlgolia:", e)
+    Sentry.captureException(e)
+    await Sentry.close()
     process.exit(1)
 })
-
-void indexExplorerViewsToAlgolia()
diff --git a/baker/algolia/indexPagesToAlgolia.tsx b/baker/algolia/indexPagesToAlgolia.tsx
@@ -1,3 +1,8 @@
+// This should be imported as early as possible so the global error handler is
+// set up before any errors are thrown.
+import "../../serverUtils/instrument.js"
+
+import * as Sentry from "@sentry/node"
 import * as db from "../../db/db.js"
 import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js"
 import { getAlgoliaClient } from "./configureAlgolia.js"
@@ -25,9 +30,9 @@ const indexPagesToAlgolia = async () => {
     process.exit(0)
 }
 
-process.on("unhandledRejection", (e) => {
-    console.error(e)
+indexPagesToAlgolia().catch(async (e) => {
+    console.error("Error in indexPagesToAlgolia:", e)
+    Sentry.captureException(e)
+    await Sentry.close()
     process.exit(1)
 })
-
-void indexPagesToAlgolia()
diff --git a/baker/algolia/utils/explorerViews.ts b/baker/algolia/utils/explorerViews.ts
@@ -470,14 +470,13 @@ async function enrichWithTableData(
     return enrichedRecords
 }
 
-function enrichRecordWithIndicatorData(
+async function enrichRecordWithIndicatorData(
     record: IndicatorUnenrichedExplorerViewRecord,
     indicatorMetadataDictionary: ExplorerIndicatorMetadataDictionary
-): IndicatorEnrichedExplorerViewRecord {
-    const allEntityNames = at(
-        indicatorMetadataDictionary,
-        record.yVariableIds
-    ).flatMap((meta) => meta.entityNames)
+): Promise<IndicatorEnrichedExplorerViewRecord | undefined> {
+    const allEntityNames = at(indicatorMetadataDictionary, record.yVariableIds)
+        .filter(Boolean)
+        .flatMap((meta) => meta.entityNames)
 
     const uniqueNonEmptyEntityNames = uniq(allEntityNames).filter(
         (name): name is string => !!name
@@ -486,6 +485,13 @@ function enrichRecordWithIndicatorData(
     const firstYIndicator = record.yVariableIds[0]
 
     const indicatorInfo = indicatorMetadataDictionary[firstYIndicator]
+    if (!indicatorInfo) {
+        await logErrorAndMaybeCaptureInSentry({
+            name: "ExplorerViewIndicatorMissing",
+            message: `Explorer with slug "${record.explorerSlug}" has a view with missing indicator metadata: ${record.viewQueryParams}.`,
+        })
+        return
+    }
 
     const viewTitle =
         record.viewTitle ||
@@ -505,16 +511,16 @@ function enrichRecordWithIndicatorData(
     }
 }
 
-const enrichWithIndicatorMetadata = async (
+async function enrichWithIndicatorMetadata(
     indicatorBaseRecords: IndicatorUnenrichedExplorerViewRecord[],
     indicatorMetadataDictionary: ExplorerIndicatorMetadataDictionary
-): Promise<IndicatorEnrichedExplorerViewRecord[]> => {
-    return indicatorBaseRecords.map((indicatorBaseRecord) =>
+): Promise<IndicatorEnrichedExplorerViewRecord[]> {
+    return pMap(indicatorBaseRecords, (indicatorBaseRecord) =>
         enrichRecordWithIndicatorData(
             indicatorBaseRecord,
             indicatorMetadataDictionary
         )
-    )
+    ).then((r) => r.filter(Boolean) as IndicatorEnrichedExplorerViewRecord[])
 }
 
 function processSubtitles(
@@ -658,6 +664,7 @@ export const getExplorerViewRecordsForExplorer = async (
         indicatorBaseRecords,
         trx
     )
+
     console.log("Fetched indicator metadata for explorer", slug)
 
     const enrichedIndicatorRecords = await enrichWithIndicatorMetadata(