From 6ffa4380dc76b1c893361f4ecd9343ef2f459ce3 Mon Sep 17 00:00:00 2001 From: Jiri Lojda Date: Wed, 31 Jan 2024 18:26:02 +0100 Subject: [PATCH] Avoid loading whole zip at once during import - avoids problems when importing a big .zip file - replaces jszip with node-stream-zip for reading zips as it supports reading only parts of the zip at a time - since jszip was replaced with archiver for writing zips (because it supports streaming), jszip is no longer needed --- package-lock.json | 48 +++++-------------- package.json | 3 +- src/commands/import.ts | 9 ++-- .../importExportEntities/entities/assets.ts | 22 +++------ .../importExportEntities/entityDefinition.ts | 6 +-- .../integration/importExport/utils/envData.ts | 11 +++-- 6 files changed, 35 insertions(+), 64 deletions(-) diff --git a/package-lock.json b/package-lock.json index a8275d11..bfbf8c9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@kontent-ai/management-sdk": "^5.8.0", "archiver": "^6.0.1", "chalk": "^5.3.0", - "jszip": "^3.10.1", + "node-stream-zip": "^1.15.0", "yargs": "^17.7.2" }, "bin": { @@ -4662,11 +4662,6 @@ "node": ">= 4" } }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" - }, "node_modules/import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", @@ -6586,17 +6581,6 @@ "node": ">=4.0" } }, - "node_modules/jszip": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", - "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", - "dependencies": { - "lie": "~3.3.0", - "pako": "~1.0.2", - "readable-stream": "~2.3.6", - "setimmediate": "^1.0.5" - } - }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -6650,14 +6634,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "dependencies": { - "immediate": "~3.0.5" - } - }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -6867,6 +6843,18 @@ "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", "dev": true }, + "node_modules/node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==", + "engines": { + "node": ">=0.12.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/antelle" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -7095,11 +7083,6 @@ "node": ">=6" } }, - "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7695,11 +7678,6 @@ "node": ">= 0.4" } }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", diff --git a/package.json b/package.json index 0962b26c..8ac6d8ab 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,6 @@ "fmt:check": "dprint check", "start": "npm run build && node ./build/src/index.js", "test:integration": "npm run build && jest --config=jestIntegrationTests.config.ts", - "debugRun": "npm run build && ./build/src/index.js export --configFile tmpConfig.json", "test:unit": "jest --config=jestUnitTests.config.ts" }, "files": [ @@ -38,7 +37,7 @@ "@kontent-ai/management-sdk": "^5.8.0", "archiver": "^6.0.1", "chalk": "^5.3.0", - "jszip": "^3.10.1", + "node-stream-zip": "^1.15.0", "yargs": "^17.7.2" }, "devDependencies": { diff --git a/src/commands/import.ts b/src/commands/import.ts index cd316bd5..3d214f49 100644 --- a/src/commands/import.ts +++ b/src/commands/import.ts @@ -1,7 +1,6 @@ import { ManagementClient } from "@kontent-ai/management-sdk"; import chalk from "chalk"; -import * as fsPromises from "fs/promises"; -import JSZip from "jszip"; +import StreamZip from "node-stream-zip"; import { logError, logInfo, LogOptions } from "../log.js"; import { RegisterCommand } from "../types/yargs.js"; @@ -101,7 +100,7 @@ type ImportEntitiesParams = & LogOptions; const importEntities = async (params: ImportEntitiesParams) => { - const root = await fsPromises.readFile(params.fileName).then(JSZip.loadAsync); + const root = new StreamZip.async({ file: params.fileName }); const client = new ManagementClient({ environmentId: params.environmentId, apiKey: params.apiKey, @@ -125,8 +124,8 @@ const importEntities = async (params: ImportEntitiesParams) => { logInfo(params, "standard", `Importing: ${chalk.yellow(def.name)}`); try { - context = await root.file(`${def.name}.json`) - ?.async("string") + context = await root.entryData(`${def.name}.json`) + .then(b => b.toString("utf8")) .then(def.deserializeEntities) .then(e => def.importEntities(client, e, context, params, root)) ?? context; diff --git a/src/commands/importExportEntities/entities/assets.ts b/src/commands/importExportEntities/entities/assets.ts index eafd7a77..ad63cb18 100644 --- a/src/commands/importExportEntities/entities/assets.ts +++ b/src/commands/importExportEntities/entities/assets.ts @@ -1,7 +1,7 @@ import { AssetContracts, ManagementClient } from "@kontent-ai/management-sdk"; import archiver from "archiver"; import chalk from "chalk"; -import JSZip from "jszip"; +import { StreamZipAsync } from "node-stream-zip"; import stream from "stream"; import { logInfo, LogOptions } from "../../../log.js"; @@ -11,7 +11,8 @@ import { getRequired } from "../../import/utils.js"; import { EntityDefinition, ImportContext } from "../entityDefinition.js"; const assetsBinariesFolderName = "assets"; -const createFileName = (asset: AssetContracts.IAssetModelContract) => `${asset.id}-${asset.file_name}`; +const createFileName = (asset: AssetContracts.IAssetModelContract) => + `${assetsBinariesFolderName}/${asset.id}-${asset.file_name}`; type AssetWithElements = FixReferences & { readonly elements: ReadonlyArray; @@ -36,18 +37,12 @@ export const assetsEntity: EntityDefinition> = ); } - const assetsZip = zip.folder(assetsBinariesFolderName); if (!fileAssets.length) { return; } - if (!assetsZip) { - throw new Error( - `It is not possible to import assets, because the folder with asset binaries ("${assetsBinariesFolderName}") is missing.`, - ); - } const assetIdEntries = await serially( - fileAssets.map(createImportAssetFetcher(assetsZip, client, context, logOptions)), + fileAssets.map(createImportAssetFetcher(zip, client, context, logOptions)), ); return { @@ -64,18 +59,15 @@ const saveAsset = async ( ) => { logInfo(logOptions, "verbose", `Exporting: file ${chalk.yellow(asset.file_name)}.`); const file = await fetch(asset.url).then(res => res.blob()).then(res => res.stream()); - archive.append(stream.Readable.fromWeb(file), { name: "assets/" + createFileName(asset) }); + archive.append(stream.Readable.fromWeb(file), { name: createFileName(asset) }); }; const createImportAssetFetcher = - (zip: JSZip, client: ManagementClient, context: ImportContext, logOptions: LogOptions) => + (zip: StreamZipAsync, client: ManagementClient, context: ImportContext, logOptions: LogOptions) => (fileAsset: AssetWithElements) => async (): Promise => { - const binary = await zip.file(createFileName(fileAsset))?.async("nodebuffer"); + const binary = await zip.entryData(createFileName(fileAsset)); - if (!binary) { - throw new Error(`Failed to load a binary file "${fileAsset.file_name}" for asset "${fileAsset.id}".`); - } const folderId = fileAsset.folder?.id ? getRequired(context.assetFolderIdsByOldIds, fileAsset.folder.id, "folder") : undefined; diff --git a/src/commands/importExportEntities/entityDefinition.ts b/src/commands/importExportEntities/entityDefinition.ts index 15f0588b..b67417d1 100644 --- a/src/commands/importExportEntities/entityDefinition.ts +++ b/src/commands/importExportEntities/entityDefinition.ts @@ -1,6 +1,6 @@ import { ManagementClient } from "@kontent-ai/management-sdk"; import archiver from "archiver"; -import JSZip from "jszip"; +import { StreamZipAsync } from "node-stream-zip"; import { LogOptions } from "../../log.js"; @@ -22,8 +22,8 @@ export type EntityImportDefinition = Readonly<{ entities: T, context: ImportContext, logOptions: LogOptions, - zip: JSZip, - ) => Promise; + zip: StreamZipAsync, + ) => Promise; }>; export type DependentImportAction = Readonly<{ diff --git a/tests/integration/importExport/utils/envData.ts b/tests/integration/importExport/utils/envData.ts index 292ab3c9..86644257 100644 --- a/tests/integration/importExport/utils/envData.ts +++ b/tests/integration/importExport/utils/envData.ts @@ -15,8 +15,7 @@ import { WorkflowContracts, } from "@kontent-ai/management-sdk"; import { config as dotenvConfig } from "dotenv"; -import * as fsPromises from "fs/promises"; -import JSZip from "jszip"; +import StreamZip, { StreamZipAsync } from "node-stream-zip"; import { serially } from "../../../../src/utils/requests"; @@ -114,7 +113,7 @@ const loadAllData = async (client: ManagementClient): Promise => ({ }); export const loadAllEnvDataFromZip = async (fileName: string): Promise => { - const zip = await fsPromises.readFile(fileName).then(buffer => JSZip.loadAsync(buffer)); + const zip = new StreamZip.async({ file: fileName }); return { collections: await loadFile(zip, "collections.json"), @@ -133,4 +132,8 @@ export const loadAllEnvDataFromZip = async (fileName: string): Promise zip.file(fileName)?.async("string").then(JSON.parse); +const loadFile = (zip: StreamZipAsync, fileName: string) => + zip.entryData(fileName) + .then(b => b.toString("utf8")) + .then(JSON.parse) + .catch(() => undefined);