Add catalogPath to mdims

* Catalog path will now be the main ID used by ETL * It's temporarily allowed to be null; we'll disallow that once existing data is migrated * Slug is now allowed to be null, since it will be only set in admin at some point * For that reason we'll use the catalog path as a slug for previews of mdims in the admin, i.e. `/admin/grapher/{catalogPath}`
owid · Feb 19, 2025 · b6d81b2 · b6d81b2
1 parent e36a891
commit b6d81b2
Show file tree

Hide file tree

Showing 13 changed files with 215 additions and 93 deletions.
diff --git a/adminSiteClient/MultiDimIndexPage.tsx b/adminSiteClient/MultiDimIndexPage.tsx
@@ -28,6 +28,7 @@ import { Link } from "./Link.js"
 
 type ApiMultiDim = {
     id: number
+    catalogPath: string | null
     title: string
     slug: string
     updatedAt: string
@@ -86,21 +87,31 @@ function createColumns(
             render: (_, record) => {
                 return (
                     <a
-                        href={urljoin(
-                            BAKED_BASE_URL,
-                            `/admin/grapher/${record.slug}`
-                        )}
+                        href={
+                            record.catalogPath
+                                ? urljoin(
+                                      BAKED_BASE_URL,
+                                      `/admin/grapher/${encodeURIComponent(
+                                          record.catalogPath
+                                      )}`
+                                  )
+                                : undefined
+                        }
                         target="_blank"
                         rel="noopener"
                     >
-                        <img
-                            src={urljoin(
-                                GRAPHER_DYNAMIC_THUMBNAIL_URL,
-                                `/${record.slug}.png?imHeight=400`
-                            )}
-                            style={{ height: "140px", width: "auto" }}
-                            alt="Preview"
-                        />
+                        {record.slug ? (
+                            <img
+                                src={urljoin(
+                                    GRAPHER_DYNAMIC_THUMBNAIL_URL,
+                                    `/${record.slug}.png?imHeight=400`
+                                )}
+                                style={{ height: "140px", width: "auto" }}
+                                alt="Preview"
+                            />
+                        ) : (
+                            "Preview"
+                        )}
                     </a>
                 )
             },
@@ -109,7 +120,6 @@ function createColumns(
             title: "Title",
             dataIndex: "title",
             key: "title",
-            width: 280,
             render: (text, record) =>
                 record.published ? (
                     <a
@@ -124,6 +134,20 @@ function createColumns(
                 ),
             sorter: (a, b) => a.title.localeCompare(b.title),
         },
+        {
+            title: "Catalog path",
+            dataIndex: "catalogPath",
+            key: "catalogPath",
+            render: (catalogPath) =>
+                catalogPath && (
+                    <Typography.Text copyable>{catalogPath}</Typography.Text>
+                ),
+            sorter: (a, b) => {
+                if (a.catalogPath === null) return 1
+                if (b.catalogPath === null) return -1
+                return a.catalogPath.localeCompare(b.catalogPath)
+            },
+        },
         {
             title: "Slug",
             dataIndex: "slug",
@@ -164,8 +188,9 @@ function createColumns(
                     <Switch
                         checked={published}
                         disabled={
-                            publishMutation.isLoading &&
-                            publishMutation.variables?.id === record.id
+                            !record.slug ||
+                            (publishMutation.isLoading &&
+                                publishMutation.variables?.id === record.id)
                         }
                     />
                 </Popconfirm>

diff --git a/adminSiteServer/adminRouter.tsx b/adminSiteServer/adminRouter.tsx
@@ -44,7 +44,10 @@ import { getChartConfigBySlug } from "../db/model/Chart.js"
 import { getVariableMetadata } from "../db/model/Variable.js"
 import { DbPlainDatasetFile, DbPlainDataset } from "@ourworldindata/types"
 import { getPlainRouteWithROTransaction } from "./plainRouterHelpers.js"
-import { getMultiDimDataPageBySlug } from "../db/model/MultiDimDataPage.js"
+import {
+    getMultiDimDataPageByCatalogPath,
+    getMultiDimDataPageBySlug,
+} from "../db/model/MultiDimDataPage.js"
 import { renderMultiDimDataPageFromConfig } from "../baker/MultiDimBaker.js"
 
 // eslint-disable-next-line @typescript-eslint/no-require-imports
@@ -356,13 +359,14 @@ getPlainRouteWithROTransaction(
             return
         }
 
-        const mdd = await getMultiDimDataPageBySlug(trx, slug, {
-            onlyPublished: false,
-        })
+        const mdd =
+            (await getMultiDimDataPageBySlug(trx, slug, {
+                onlyPublished: false,
+            })) ?? (await getMultiDimDataPageByCatalogPath(trx, slug))
         if (mdd) {
             const renderedPage = await renderMultiDimDataPageFromConfig({
                 knex: trx,
-                slug,
+                slug: mdd.slug,
                 config: mdd.config,
                 isPreviewing: true,
             })

diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts
@@ -269,7 +269,11 @@ getRouteWithROTransaction(apiRouter, "/images/usage", getImageUsageHandler)
 
 // Mdim routes
 getRouteWithROTransaction(apiRouter, "/multi-dims.json", handleGetMultiDims)
-putRouteWithRWTransaction(apiRouter, "/multi-dim/:slug", handlePutMultiDim)
+putRouteWithRWTransaction(
+    apiRouter,
+    "/multi-dims/:catalogPath",
+    handlePutMultiDim
+)
 patchRouteWithRWTransaction(apiRouter, "/multi-dims/:id", handlePatchMultiDim)
 
 // Misc routes

diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts
@@ -1,22 +1,24 @@
 import {
+    DbPlainMultiDimDataPage,
     JsonError,
     MultiDimDataPageConfigRaw,
     MultiDimDataPagesTableName,
 } from "@ourworldindata/types"
+import { getMultiDimDataPageById } from "../../db/model/MultiDimDataPage.js"
+import { expectInt } from "../../serverUtils/serverUtil.js"
 import {
-    getMultiDimDataPageById,
-    multiDimDataPageExists,
-} from "../../db/model/MultiDimDataPage.js"
-import { expectInt, isValidSlug } from "../../serverUtils/serverUtil.js"
-import {
-    upsertMultiDimConfig,
+    upsertMultiDim,
     setMultiDimPublished,
     setMultiDimSlug,
 } from "../multiDim.js"
 import { triggerStaticBuild } from "./routeUtils.js"
 import { Request } from "../authentication.js"
 import * as db from "../../db/db.js"
-import { validateNewGrapherSlug, validateMultiDimSlug } from "../validation.js"
+import {
+    validateNewGrapherSlug,
+    validateMultiDimSlug,
+    isValidCatalogPath,
+} from "../validation.js"
 import e from "express"
 
 export async function handleGetMultiDims(
@@ -25,17 +27,17 @@ export async function handleGetMultiDims(
     trx: db.KnexReadonlyTransaction
 ) {
     try {
-        const results = await db.knexRaw<{
-            id: number
-            slug: string
-            title: string
-            updatedAt: string
-            published: number
-        }>(
+        const results = await db.knexRaw<
+            Pick<
+                DbPlainMultiDimDataPage,
+                "id" | "catalogPath" | "slug" | "updatedAt" | "published"
+            > & { title: string }
+        >(
             trx,
             `-- sql
             SELECT
                 id,
+                catalogPath,
                 slug,
                 config->>'$.title.title' as title,
                 updatedAt,
@@ -57,20 +59,25 @@ export async function handlePutMultiDim(
     res: e.Response<any, Record<string, any>>,
     trx: db.KnexReadWriteTransaction
 ) {
-    const { slug } = req.params
-    if (!isValidSlug(slug)) {
-        throw new JsonError(`Invalid multi-dim slug ${slug}`)
+    const { catalogPath } = req.params
+    if (!isValidCatalogPath(catalogPath)) {
+        throw new JsonError(`Invalid multi-dim catalog path ${catalogPath}`)
     }
-    if (!(await multiDimDataPageExists(trx, { slug }))) {
-        await validateNewGrapherSlug(trx, slug)
+    const { config: rawConfig } = req.body as {
+        config: MultiDimDataPageConfigRaw
     }
-    const rawConfig = req.body as MultiDimDataPageConfigRaw
-    const id = await upsertMultiDimConfig(trx, slug, rawConfig)
+    const id = await upsertMultiDim(trx, catalogPath, rawConfig)
 
-    if (await multiDimDataPageExists(trx, { slug, published: true })) {
+    const { slug: publishedSlug } =
+        (await trx<DbPlainMultiDimDataPage>(MultiDimDataPagesTableName)
+            .select("slug")
+            .where("catalogPath", catalogPath)
+            .where("published", true)
+            .first()) ?? {}
+    if (publishedSlug) {
         await triggerStaticBuild(
             res.locals.user,
-            `Publishing multidimensional chart ${slug}`
+            `Publishing multidimensional chart ${publishedSlug}`
         )
     }
     return { success: true, id }

diff --git a/adminSiteServer/multiDim.ts b/adminSiteServer/multiDim.ts
@@ -5,7 +5,6 @@ import {
     migrateGrapherConfigToLatestVersion,
 } from "@ourworldindata/grapher"
 import {
-    Base64String,
     ChartConfigsTableName,
     DbEnrichedMultiDimDataPage,
     DbPlainMultiDimDataPage,
@@ -23,6 +22,7 @@ import {
     MultiDimDimensionChoices,
     MultiDimXChartConfigsTableName,
     parseChartConfigsRow,
+    R2GrapherConfigDirectory,
     View,
 } from "@ourworldindata/types"
 import {
@@ -38,6 +38,7 @@ import {
     getVariableIdsByCatalogPath,
 } from "../db/model/Variable.js"
 import {
+    deleteGrapherConfigFromR2,
     deleteGrapherConfigFromR2ByUUID,
     saveMultiDimConfigToR2,
 } from "./chartConfigR2Helpers.js"
@@ -166,53 +167,62 @@ async function resolveMultiDimDataPageCatalogPathsToIndicatorIds(
 
 async function getViewIdToChartConfigIdMap(
     knex: db.KnexReadonlyTransaction,
-    slug: string
+    catalogPath: string
 ) {
     const rows = await db.knexRaw<DbPlainMultiDimXChartConfig>(
         knex,
         `-- sql
         SELECT viewId, chartConfigId
         FROM multi_dim_x_chart_configs mdxcc
         JOIN multi_dim_data_pages mddp ON mddp.id = mdxcc.multiDimId
-        WHERE mddp.slug = ?`,
-        [slug]
+        WHERE mddp.catalogPath = ?`,
+        [catalogPath]
     )
-    return new Map(
-        rows.map((row) => [row.viewId, row.chartConfigId as Base64String])
+    return new Map(rows.map((row) => [row.viewId, row.chartConfigId]))
+}
+
+async function retrieveMultiDimConfigFromDbAndSaveToR2(
+    knex: db.KnexReadonlyTransaction,
+    id: number
+) {
+    // We need to get the full config and the md5 hash from the database instead of
+    // computing our own md5 hash because MySQL normalizes JSON and our
+    // client computed md5 would be different from the ones computed by and stored in R2
+    const result = await knex<DbPlainMultiDimDataPage>(
+        MultiDimDataPagesTableName
     )
+        .select("slug", "config", "configMd5")
+        .where({ id })
+        .first()
+    const { slug, config: normalizedConfig, configMd5 } = result!
+    if (slug) {
+        await saveMultiDimConfigToR2(normalizedConfig, slug, configMd5)
+    }
 }
 
-async function saveMultiDimConfig(
+async function upsertMultiDimConfig(
     knex: db.KnexReadWriteTransaction,
-    slug: string,
+    catalogPath: string,
     config: MultiDimDataPageConfigEnriched
 ) {
     const id = await upsertMultiDimDataPage(knex, {
-        slug,
+        catalogPath,
         config: JSON.stringify(config),
     })
     if (id === 0) {
         // There are no updates to the config, return the existing id.
-        console.debug(`There are no changes to multi dim config slug=${slug}`)
+        console.debug(
+            `There are no changes to multi dim config catalogPath=${catalogPath}`
+        )
         const result = await knex<DbPlainMultiDimDataPage>(
             MultiDimDataPagesTableName
         )
             .select("id")
-            .where({ slug })
+            .where({ catalogPath })
             .first()
         return result!.id
     }
-    // We need to get the full config and the md5 hash from the database instead of
-    // computing our own md5 hash because MySQL normalizes JSON and our
-    // client computed md5 would be different from the ones computed by and stored in R2
-    const result = await knex<DbPlainMultiDimDataPage>(
-        MultiDimDataPagesTableName
-    )
-        .select("config", "configMd5")
-        .where({ id })
-        .first()
-    const { config: normalizedConfig, configMd5 } = result!
-    await saveMultiDimConfigToR2(normalizedConfig, slug, configMd5)
+    await retrieveMultiDimConfigFromDbAndSaveToR2(knex, id)
     return id
 }
 
@@ -231,9 +241,9 @@ async function cleanUpOrphanedChartConfigs(
     }
 }
 
-export async function upsertMultiDimConfig(
+export async function upsertMultiDim(
     knex: db.KnexReadWriteTransaction,
-    slug: string,
+    catalogPath: string,
     rawConfig: MultiDimDataPageConfigRaw
 ): Promise<number> {
     const config = await resolveMultiDimDataPageCatalogPathsToIndicatorIds(
@@ -246,7 +256,7 @@ export async function upsertMultiDimConfig(
     )
     const existingViewIdsToChartConfigIds = await getViewIdToChartConfigIdMap(
         knex,
-        slug
+        catalogPath
     )
     const reusedChartConfigIds = new Set<string>()
     const { grapherConfigSchema } = config
@@ -259,7 +269,6 @@ export async function upsertMultiDimConfig(
                 $schema: defaultGrapherConfig.$schema,
                 dimensions: MultiDimDataPageConfig.viewToDimensionsConfig(view),
                 selectedEntityNames: config.defaultSelection ?? [],
-                slug,
             }
             let viewGrapherConfig = {}
             if (view.config) {
@@ -313,7 +322,11 @@ export async function upsertMultiDimConfig(
     await cleanUpOrphanedChartConfigs(knex, orphanedChartConfigIds)
 
     const enrichedConfig = { ...config, views: enrichedViews }
-    const multiDimId = await saveMultiDimConfig(knex, slug, enrichedConfig)
+    const multiDimId = await upsertMultiDimConfig(
+        knex,
+        catalogPath,
+        enrichedConfig
+    )
     for (const view of enrichedConfig.views) {
         await upsertMultiDimXChartConfigs(knex, {
             multiDimId,
@@ -376,5 +389,10 @@ export async function setMultiDimSlug(
     await knex(MultiDimDataPagesTableName)
         .where({ id: multiDim.id })
         .update({ slug })
+    await deleteGrapherConfigFromR2(
+        R2GrapherConfigDirectory.multiDim,
+        `${multiDim.slug}.json`
+    )
+    await retrieveMultiDimConfigFromDbAndSaveToR2(knex, multiDim.id)
     return { ...multiDim, slug }
 }