Avoid loading whole zip at once during import

- avoids problems when importing a big .zip file
- replaces jszip with node-stream-zip for reading zips as it supports reading only parts of the zip at a time
- since jszip was replaced with archiver for writing zips (because it supports streaming), jszip is no longer needed

package.json

@@ -10,7 +10,6 @@
     "fmt:check": "dprint check",
     "start": "npm run build && node ./build/src/index.js",
     "test:integration": "npm run build && jest --config=jestIntegrationTests.config.ts",
-    "debugRun": "npm run build && ./build/src/index.js export --configFile tmpConfig.json",
     "test:unit": "jest --config=jestUnitTests.config.ts"
   },
   "files": [
@@ -38,7 +37,7 @@
     "@kontent-ai/management-sdk": "^5.8.0",
     "archiver": "^6.0.1",
     "chalk": "^5.3.0",
-    "jszip": "^3.10.1",
+    "node-stream-zip": "^1.15.0",
     "yargs": "^17.7.2"
   },
   "devDependencies": {

src/commands/import.ts

@@ -1,7 +1,6 @@
 import { ManagementClient } from "@kontent-ai/management-sdk";
 import chalk from "chalk";
-import * as fsPromises from "fs/promises";
-import JSZip from "jszip";
+import StreamZip from "node-stream-zip";
 
 import { logError, logInfo, LogOptions } from "../log.js";
 import { RegisterCommand } from "../types/yargs.js";
@@ -101,7 +100,7 @@ type ImportEntitiesParams =
   & LogOptions;
 
 const importEntities = async (params: ImportEntitiesParams) => {
-  const root = await fsPromises.readFile(params.fileName).then(JSZip.loadAsync);
+  const root = new StreamZip.async({ file: params.fileName });
   const client = new ManagementClient({
     environmentId: params.environmentId,
     apiKey: params.apiKey,
@@ -125,8 +124,8 @@ const importEntities = async (params: ImportEntitiesParams) => {
     logInfo(params, "standard", `Importing: ${chalk.yellow(def.name)}`);
 
     try {
-      context = await root.file(`${def.name}.json`)
-        ?.async("string")
+      context = await root.entryData(`${def.name}.json`)
+        .then(b => b.toString("utf8"))
         .then(def.deserializeEntities)
         .then(e => def.importEntities(client, e, context, params, root))
         ?? context;

src/commands/importExportEntities/entities/assets.ts

@@ -1,7 +1,7 @@
 import { AssetContracts, ManagementClient } from "@kontent-ai/management-sdk";
 import archiver from "archiver";
 import chalk from "chalk";
-import JSZip from "jszip";
+import { StreamZipAsync } from "node-stream-zip";
 import stream from "stream";
 
 import { logInfo, LogOptions } from "../../../log.js";
@@ -11,7 +11,8 @@ import { getRequired } from "../../import/utils.js";
 import { EntityDefinition, ImportContext } from "../entityDefinition.js";
 
 const assetsBinariesFolderName = "assets";
-const createFileName = (asset: AssetContracts.IAssetModelContract) => `${asset.id}-${asset.file_name}`;
+const createFileName = (asset: AssetContracts.IAssetModelContract) =>
+  `${assetsBinariesFolderName}/${asset.id}-${asset.file_name}`;
 
 type AssetWithElements = FixReferences<AssetContracts.IAssetModelContract> & {
   readonly elements: ReadonlyArray<unknown>;
@@ -36,18 +37,12 @@ export const assetsEntity: EntityDefinition<ReadonlyArray<AssetWithElements>> =
       );
     }
 
-    const assetsZip = zip.folder(assetsBinariesFolderName);
     if (!fileAssets.length) {
       return;
     }
-    if (!assetsZip) {
-      throw new Error(
-        `It is not possible to import assets, because the folder with asset binaries ("${assetsBinariesFolderName}") is missing.`,
-      );
-    }
 
     const assetIdEntries = await serially(
-      fileAssets.map(createImportAssetFetcher(assetsZip, client, context, logOptions)),
+      fileAssets.map(createImportAssetFetcher(zip, client, context, logOptions)),
     );
 
     return {
@@ -64,18 +59,15 @@ const saveAsset = async (
 ) => {
   logInfo(logOptions, "verbose", `Exporting: file ${chalk.yellow(asset.file_name)}.`);
   const file = await fetch(asset.url).then(res => res.blob()).then(res => res.stream());
-  archive.append(stream.Readable.fromWeb(file), { name: "assets/" + createFileName(asset) });
+  archive.append(stream.Readable.fromWeb(file), { name: createFileName(asset) });
 };
 
 const createImportAssetFetcher =
-  (zip: JSZip, client: ManagementClient, context: ImportContext, logOptions: LogOptions) =>
+  (zip: StreamZipAsync, client: ManagementClient, context: ImportContext, logOptions: LogOptions) =>
   (fileAsset: AssetWithElements) =>
   async (): Promise<readonly [string, string]> => {
-    const binary = await zip.file(createFileName(fileAsset))?.async("nodebuffer");
+    const binary = await zip.entryData(createFileName(fileAsset));
 
-    if (!binary) {
-      throw new Error(`Failed to load a binary file "${fileAsset.file_name}" for asset "${fileAsset.id}".`);
-    }
     const folderId = fileAsset.folder?.id
       ? getRequired(context.assetFolderIdsByOldIds, fileAsset.folder.id, "folder")
       : undefined;

src/commands/importExportEntities/entityDefinition.ts

@@ -1,6 +1,6 @@
 import { ManagementClient } from "@kontent-ai/management-sdk";
 import archiver from "archiver";
-import JSZip from "jszip";
+import { StreamZipAsync } from "node-stream-zip";
 
 import { LogOptions } from "../../log.js";
 
@@ -22,8 +22,8 @@ export type EntityImportDefinition<T> = Readonly<{
     entities: T,
     context: ImportContext,
     logOptions: LogOptions,
-    zip: JSZip,
-  ) => Promise<void | ImportContext>;
+    zip: StreamZipAsync,
+  ) => Promise<void | undefined | ImportContext>;
 }>;
 
 export type DependentImportAction<T> = Readonly<{

tests/integration/importExport/utils/envData.ts

@@ -15,8 +15,7 @@ import {
   WorkflowContracts,
 } from "@kontent-ai/management-sdk";
 import { config as dotenvConfig } from "dotenv";
-import * as fsPromises from "fs/promises";
-import JSZip from "jszip";
+import StreamZip, { StreamZipAsync } from "node-stream-zip";
 
 import { serially } from "../../../../src/utils/requests";
 
@@ -114,7 +113,7 @@ const loadAllData = async (client: ManagementClient): Promise<AllEnvData> => ({
 });
 
 export const loadAllEnvDataFromZip = async (fileName: string): Promise<AllEnvData> => {
-  const zip = await fsPromises.readFile(fileName).then(buffer => JSZip.loadAsync(buffer));
+  const zip = new StreamZip.async({ file: fileName });
 
   return {
     collections: await loadFile(zip, "collections.json"),
@@ -133,4 +132,8 @@ export const loadAllEnvDataFromZip = async (fileName: string): Promise<AllEnvDat
   };
 };
 
-const loadFile = (zip: JSZip, fileName: string) => zip.file(fileName)?.async("string").then(JSON.parse);
+const loadFile = (zip: StreamZipAsync, fileName: string) =>
+  zip.entryData(fileName)
+    .then(b => b.toString("utf8"))
+    .then(JSON.parse)
+    .catch(() => undefined);