Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Marcel's enhancements to server-side csv generation #4051

Merged
merged 6 commits into from
Oct 10, 2024
Merged

Marcel's enhancements to server-side csv generation #4051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9d78097
enhance: use `Object.fromEntries()`
marcelgerber Oct 10, 2024
192cf7d
fix: use `canonicalUrl` over `originUrl`
marcelgerber Oct 10, 2024
665c438
fix: set correct content-type for readme
marcelgerber Oct 10, 2024
49eb906
chore(deps): dedupe
marcelgerber Oct 10, 2024
0615d5d
enhance: use `littlezipper` instead of `jszip`
marcelgerber Oct 10, 2024
6195555
enhance: centralize `columnsToIgnore`
marcelgerber Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 26 additions & 34 deletions functions/_common/grapherRenderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { constructReadme } from "./readmeTools"
import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm"
import { TimeLogger } from "./timeLogger"
import { png, StatusError } from "itty-router"
import JSZip from "jszip"
import { createZip, File } from "littlezipper"

import svg2png_wasm from "../../node_modules/svg2png-wasm/svg2png_wasm_bg.wasm"

Expand All @@ -27,7 +27,6 @@ import LatoMedium from "../_common/fonts/LatoLatin-Medium.ttf.bin"
import LatoBold from "../_common/fonts/LatoLatin-Bold.ttf.bin"
import PlayfairSemiBold from "../_common/fonts/PlayfairDisplayLatin-SemiBold.ttf.bin"
import { Env } from "./env.js"
import { fromPairs } from "lodash"

declare global {
// eslint-disable-next-line no-var
Expand Down Expand Up @@ -318,7 +317,7 @@ async function initGrapher(
return grapher
}

function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
const getColumnsForMetadata = (grapher: Grapher) => {
const columnsToIgnore = new Set(
[
OwidTableSlugs.entityId,
Expand All @@ -331,12 +330,19 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
].map((slug) => slug.toString())
)

const columnsToGet = grapher.inputTable.columnSlugs.filter(
const colsToGet = grapher.inputTable.columnSlugs.filter(
(col) => !columnsToIgnore.has(col)
)

return grapher.inputTable.getColumns(colsToGet)
}

function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
const useShortNames = searchParams.get("useColumnShortNames") === "true"
console.log("useShortNames", useShortNames)

const metadataCols = getColumnsForMetadata(grapher)

const columns: [
string,
{
Expand Down Expand Up @@ -367,7 +373,7 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
>[]
shortName: string
},
][] = grapher.inputTable.getColumns(columnsToGet).map((col) => {
][] = metadataCols.map((col) => {
console.log("mapping col", col.name)
const {
descriptionShort,
Expand Down Expand Up @@ -498,7 +504,7 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
originalChartUrl: grapher.canonicalUrl,
selection: grapher.selectedEntityNames,
},
columns: fromPairs(columns),
columns: Object.fromEntries(columns),
}

return fullMetadata
Expand Down Expand Up @@ -545,14 +551,16 @@ export async function fetchZipForGrapher(
const readme = assembleReadme(grapher)
const csv = assembleCsv(grapher, searchParams)
console.log("Fetched the parts, creating zip file")
const zip = new JSZip()
zip.file(
`${identifier.id}.metadata.json`,
JSON.stringify(metadata, undefined, 2)
)
zip.file(`${identifier.id}.csv`, csv)
zip.file("readme.md", readme)
const content = await zip.generateAsync({ type: "arraybuffer" })

const zipContent: File[] = [
{
path: `${identifier.id}.metadata.json`,
data: JSON.stringify(metadata, undefined, 2),
},
{ path: `${identifier.id}.csv`, data: csv },
{ path: "readme.md", data: readme },
]
const content = await createZip(zipContent)
console.log("Generated content, returning response")
return new Response(content, {
headers: {
Expand Down Expand Up @@ -623,32 +631,16 @@ export async function fetchReadmeForGrapher(
const readme = assembleReadme(grapher)
return new Response(readme, {
headers: {
"Content-Type": "text/markdown",
"Content-Type": "text/markdown; charset=utf-8",
},
})
}

function assembleReadme(grapher: Grapher): string {
const columnsToIgnore = new Set(
[
OwidTableSlugs.entityId,
OwidTableSlugs.time,
OwidTableSlugs.entityColor,
OwidTableSlugs.entityName,
OwidTableSlugs.entityCode,
OwidTableSlugs.year,
OwidTableSlugs.day,
].map((slug) => slug.toString())
)

const columnsToGet = grapher.inputTable.columnSlugs.filter(
(col) => !columnsToIgnore.has(col)
)

const columns = grapher.inputTable.getColumns(columnsToGet)

return constructReadme(grapher, columns)
const metadataCols = getColumnsForMetadata(grapher)
return constructReadme(grapher, metadataCols)
}

async function fetchAndRenderGrapherToSvg(
identifier: GrapherIdentifier,
options: ImageOptions,
Expand Down
4 changes: 2 additions & 2 deletions functions/_common/readmeTools.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ export function constructReadme(
if (isSingleColumn)
readme = `# ${grapher.title} - Data package

This data package contains the data that powers the chart ["${grapher.title}"](${grapher.originUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}.
This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}.

## CSV Structure

Expand Down Expand Up @@ -274,7 +274,7 @@ ${sources.join("\n")}
else
readme = `# ${grapher.title} - Data package

This data package contains the data that powers the chart ["${grapher.title}"](${grapher.originUrl}) on the Our World in Data website.
This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website.

## CSV Structure

Expand Down
2 changes: 1 addition & 1 deletion functions/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"@ourworldindata/grapher": "workspace:^",
"@ourworldindata/utils": "workspace:^",
"itty-router": "^5.0.17",
"jszip": "^3.10.1",
"littlezipper": "^0.1.4",
"stripe": "^14.20.0",
"svg2png-wasm": "^1.4.1"
},
Expand Down
91 changes: 11 additions & 80 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2938,7 +2938,7 @@ __metadata:
languageName: node
linkType: hard

"@npmcli/package-json@npm:5.2.0, @npmcli/package-json@npm:^5.1.0":
"@npmcli/package-json@npm:5.2.0, @npmcli/package-json@npm:^5.0.0, @npmcli/package-json@npm:^5.1.0":
version: 5.2.0
resolution: "@npmcli/package-json@npm:5.2.0"
dependencies:
Expand All @@ -2953,21 +2953,6 @@ __metadata:
languageName: node
linkType: hard

"@npmcli/package-json@npm:^5.0.0":
version: 5.1.0
resolution: "@npmcli/package-json@npm:5.1.0"
dependencies:
"@npmcli/git": "npm:^5.0.0"
glob: "npm:^10.2.2"
hosted-git-info: "npm:^7.0.0"
json-parse-even-better-errors: "npm:^3.0.0"
normalize-package-data: "npm:^6.0.0"
proc-log: "npm:^4.0.0"
semver: "npm:^7.5.3"
checksum: 10/0e5cb5eff32cf80234525160a702c91a38e4b98ab74e34e2632b43c4350dbad170bd835989cc7d6e18d24798e3242e45b60f3d5e26bd128fe1c4529931105f8e
languageName: node
linkType: hard

"@npmcli/promise-spawn@npm:^7.0.0":
version: 7.0.2
resolution: "@npmcli/promise-spawn@npm:7.0.2"
Expand Down Expand Up @@ -8556,7 +8541,7 @@ __metadata:
languageName: node
linkType: hard

"dedent@npm:1.5.3":
"dedent@npm:1.5.3, dedent@npm:^1.0.0":
version: 1.5.3
resolution: "dedent@npm:1.5.3"
peerDependencies:
Expand All @@ -8568,18 +8553,6 @@ __metadata:
languageName: node
linkType: hard

"dedent@npm:^1.0.0":
version: 1.5.1
resolution: "dedent@npm:1.5.1"
peerDependencies:
babel-plugin-macros: ^3.1.0
peerDependenciesMeta:
babel-plugin-macros:
optional: true
checksum: 10/fc00a8bc3dfb7c413a778dc40ee8151b6c6ff35159d641f36ecd839c1df5c6e0ec5f4992e658c82624a1a62aaecaffc23b9c965ceb0bbf4d698bfc16469ac27d
languageName: node
linkType: hard

"deep-extend@npm:^0.6.0":
version: 0.6.0
resolution: "deep-extend@npm:0.6.0"
Expand Down Expand Up @@ -8853,14 +8826,7 @@ __metadata:
languageName: node
linkType: hard

"dotenv@npm:^16.0.1, dotenv@npm:^16.0.3":
version: 16.3.1
resolution: "dotenv@npm:16.3.1"
checksum: 10/dbb778237ef8750e9e3cd1473d3c8eaa9cc3600e33a75c0e36415d0fa0848197f56c3800f77924c70e7828f0b03896818cd52f785b07b9ad4d88dba73fbba83f
languageName: node
linkType: hard

"dotenv@npm:^16.4.4, dotenv@npm:~16.4.5":
"dotenv@npm:^16.0.1, dotenv@npm:^16.0.3, dotenv@npm:^16.4.4, dotenv@npm:~16.4.5":
version: 16.4.5
resolution: "dotenv@npm:16.4.5"
checksum: 10/55a3134601115194ae0f924e54473459ed0d9fc340ae610b676e248cca45aa7c680d86365318ea964e6da4e2ea80c4514c1adab5adb43d6867fb57ff068f95c8
Expand Down Expand Up @@ -11617,13 +11583,6 @@ __metadata:
languageName: node
linkType: hard

"immediate@npm:~3.0.5":
version: 3.0.6
resolution: "immediate@npm:3.0.6"
checksum: 10/f9b3486477555997657f70318cc8d3416159f208bec4cca3ff3442fd266bc23f50f0c9bd8547e1371a6b5e82b821ec9a7044a4f7b944798b25aa3cc6d5e63e62
languageName: node
linkType: hard

"immutable@npm:^4.0.0, immutable@npm:^4.3.6":
version: 4.3.6
resolution: "immutable@npm:4.3.6"
Expand Down Expand Up @@ -13243,18 +13202,6 @@ __metadata:
languageName: node
linkType: hard

"jszip@npm:^3.10.1":
version: 3.10.1
resolution: "jszip@npm:3.10.1"
dependencies:
lie: "npm:~3.3.0"
pako: "npm:~1.0.2"
readable-stream: "npm:~2.3.6"
setimmediate: "npm:^1.0.5"
checksum: 10/bfbfbb9b0a27121330ac46ab9cdb3b4812433faa9ba4a54742c87ca441e31a6194ff70ae12acefa5fe25406c432290e68003900541d948a169b23d30c34dd984
languageName: node
linkType: hard

"just-diff-apply@npm:^5.2.0":
version: 5.5.0
resolution: "just-diff-apply@npm:5.5.0"
Expand Down Expand Up @@ -13520,15 +13467,6 @@ __metadata:
languageName: node
linkType: hard

"lie@npm:~3.3.0":
version: 3.3.0
resolution: "lie@npm:3.3.0"
dependencies:
immediate: "npm:~3.0.5"
checksum: 10/f335ce67fe221af496185d7ce39c8321304adb701e122942c495f4f72dcee8803f9315ee572f5f8e8b08b9e8d7195da91b9fad776e8864746ba8b5e910adf76e
languageName: node
linkType: hard

"lilconfig@npm:3.0.0":
version: 3.0.0
resolution: "lilconfig@npm:3.0.0"
Expand Down Expand Up @@ -13612,6 +13550,13 @@ __metadata:
languageName: node
linkType: hard

"littlezipper@npm:^0.1.4":
version: 0.1.4
resolution: "littlezipper@npm:0.1.4"
checksum: 10/72b14b035a4cd2b5ac9999b955c5f17dcfa67859d83857f1c90d82a70cea4980cdc894915a8f94ed215cb5d207fcd64aa358081581349a167f7e2a261e3c5a04
languageName: node
linkType: hard

"load-json-file@npm:6.2.0":
version: 6.2.0
resolution: "load-json-file@npm:6.2.0"
Expand Down Expand Up @@ -15331,7 +15276,7 @@ __metadata:
"@ourworldindata/grapher": "workspace:^"
"@ourworldindata/utils": "workspace:^"
itty-router: "npm:^5.0.17"
jszip: "npm:^3.10.1"
littlezipper: "npm:^0.1.4"
stripe: "npm:^14.20.0"
svg2png-wasm: "npm:^1.4.1"
languageName: unknown
Expand Down Expand Up @@ -15507,13 +15452,6 @@ __metadata:
languageName: node
linkType: hard

"pako@npm:~1.0.2":
version: 1.0.11
resolution: "pako@npm:1.0.11"
checksum: 10/1ad07210e894472685564c4d39a08717e84c2a68a70d3c1d9e657d32394ef1670e22972a433cbfe48976cb98b154ba06855dcd3fcfba77f60f1777634bec48c0
languageName: node
linkType: hard

"papaparse@npm:^5.3.1":
version: 5.3.1
resolution: "papaparse@npm:5.3.1"
Expand Down Expand Up @@ -18114,13 +18052,6 @@ __metadata:
languageName: node
linkType: hard

"setimmediate@npm:^1.0.5":
version: 1.0.5
resolution: "setimmediate@npm:1.0.5"
checksum: 10/76e3f5d7f4b581b6100ff819761f04a984fa3f3990e72a6554b57188ded53efce2d3d6c0932c10f810b7c59414f85e2ab3c11521877d1dea1ce0b56dc906f485
languageName: node
linkType: hard

"setprototypeof@npm:1.2.0":
version: 1.2.0
resolution: "setprototypeof@npm:1.2.0"
Expand Down