diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 719a021a6..488e4f7b5 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -138,9 +138,9 @@ '@types/micromatch', '@types/semver', '@types/tar', - '@types/unzipper', '@types/xml2js', '@types/yargs', + '@types/yauzl', ], // Only group non-major updates matchUpdateTypes: ['patch', 'minor'] diff --git a/package-lock.json b/package-lock.json index 1299c61e4..9a69f7e14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,10 +38,10 @@ "term-size": "4.0.0", "trash": "8.1.1", "typescript-memoize": "1.1.1", - "unzipper": "0.11.4", "wrap-ansi": "8.1.0", "xml2js": "0.6.2", - "yargs": "17.7.2" + "yargs": "17.7.2", + "yauzl": "3.1.3" }, "bin": { "igir": "dist/index.js" @@ -58,10 +58,10 @@ "@types/node": "20.12.7", "@types/semver": "7.5.8", "@types/tar": "6.1.13", - "@types/unzipper": "0.10.9", "@types/which": "3.0.3", "@types/xml2js": "0.4.14", "@types/yargs": "17.0.32", + "@types/yauzl": "2.10.3", "@typescript-eslint/eslint-plugin": "7.7.1", "@typescript-eslint/parser": "7.7.1", "auto-changelog": "2.4.0", @@ -2348,15 +2348,6 @@ "minipass": "^4.0.0" } }, - "node_modules/@types/unzipper": { - "version": "0.10.9", - "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.9.tgz", - "integrity": "sha512-vHbmFZAw8emNAOVkHVbS3qBnbr0x/qHQZ+ei1HE7Oy6Tyrptl+jpqnOX+BF5owcu/HZLOV0nJK+K9sjs1Ox2JA==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/which": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/which/-/which-3.0.3.tgz", @@ -2387,6 +2378,15 @@ "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "dev": true }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.7.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.7.1.tgz", @@ -3175,14 +3175,6 @@ } ] }, - "node_modules/big-integer": { - "version": "1.6.52", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", - "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", - "engines": { - "node": ">=0.6" - } - }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -3232,11 +3224,6 @@ "node": ">= 6" } }, - "node_modules/bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==" - }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -4307,46 +4294,6 @@ "node": ">=6.0.0" } }, - "node_modules/duplexer2": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", - "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", - "dependencies": { - "readable-stream": "^2.0.2" - } - }, - "node_modules/duplexer2/node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" - }, - "node_modules/duplexer2/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/duplexer2/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/duplexer2/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -5502,81 +5449,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "dependencies": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - }, - "engines": { - "node": ">=0.6" - } - }, - "node_modules/fstream/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/fstream/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/fstream/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/fstream/node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, - "node_modules/fstream/node_modules/rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -8410,6 +8282,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -8959,6 +8832,11 @@ "node": ">=8" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -10692,18 +10570,6 @@ "node": ">= 10.0.0" } }, - "node_modules/unzipper": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.11.4.tgz", - "integrity": "sha512-T6CZQdmCMhlpHM+x4E5E9pIYCXH5INcrI8Cowr4tLQIciuw5nnp+X/LEwgeuFnay3vp9hVo4ydPw3WYSg2agWQ==", - "dependencies": { - "big-integer": "^1.6.17", - "bluebird": "~3.4.1", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2" - } - }, "node_modules/update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -11110,6 +10976,26 @@ "node": ">=12" } }, + "node_modules/yauzl": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.1.3.tgz", + "integrity": "sha512-JCCdmlJJWv7L0q/KylOekyRaUrdEoUxWkWVcgorosTROCFWiS9p2NNPE9Yb91ak7b1N5SxAZEliWpspbZccivw==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl/node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/package.json b/package.json index 981e3d9ac..d8d1cc565 100644 --- a/package.json +++ b/package.json @@ -96,10 +96,10 @@ "term-size": "4.0.0", "trash": "8.1.1", "typescript-memoize": "1.1.1", - "unzipper": "0.11.4", "wrap-ansi": "8.1.0", "xml2js": "0.6.2", - "yargs": "17.7.2" + "yargs": "17.7.2", + "yauzl": "3.1.3" }, "devDependencies": { "@jest/globals": "29.7.0", @@ -113,10 +113,10 @@ "@types/node": "20.12.7", "@types/semver": "7.5.8", "@types/tar": "6.1.13", - "@types/unzipper": "0.10.9", "@types/which": "3.0.3", "@types/xml2js": "0.4.14", "@types/yargs": "17.0.32", + "@types/yauzl": "2.10.3", "@typescript-eslint/eslint-plugin": "7.7.1", "@typescript-eslint/parser": "7.7.1", "auto-changelog": "2.4.0", diff --git a/src/types/files/archives/zip.ts b/src/types/files/archives/zip.ts index f38c007a2..71308455b 100644 --- a/src/types/files/archives/zip.ts +++ b/src/types/files/archives/zip.ts @@ -4,12 +4,11 @@ import { Readable } from 'node:stream'; import { clearInterval } from 'node:timers'; import archiver, { Archiver } from 'archiver'; -import async, { AsyncResultCallback } from 'async'; -import unzipper, { Entry } from 'unzipper'; +import async from 'async'; +import yauzl from 'yauzl'; import Constants from '../../../constants.js'; import fsPoly from '../../../polyfill/fsPoly.js'; -import StreamPoly from '../../../polyfill/streamPoly.js'; import File from '../file.js'; import FileChecksums, { ChecksumBitmask, ChecksumProps } from '../fileChecksums.js'; import Archive from './archive.js'; @@ -24,46 +23,69 @@ export default class Zip extends Archive { } async getArchiveEntries(checksumBitmask: number): Promise[]> { - // https://github.com/ZJONSSON/node-unzipper/issues/280 - // UTF-8 entry names are not decoded correctly - // But this is mitigated by `extractEntryToStream()` and therefore `extractEntryToFile()` both - // using `unzipper.Open.file()` as well, so mangled filenames here will still extract fine - const archive = await unzipper.Open.file(this.getFilePath()); - - return async.mapLimit( - archive.files.filter((entryFile) => entryFile.type === 'File'), - Constants.ARCHIVE_ENTRY_SCANNER_THREADS_PER_ARCHIVE, - async (entryFile, callback: AsyncResultCallback, Error>) => { - let checksums: ChecksumProps = {}; - if (checksumBitmask & ~ChecksumBitmask.CRC32) { - const entryStream = entryFile.stream() - // Ignore FILE_ENDED exceptions. This may cause entries to have an empty path, which - // may lead to unexpected behavior, but at least this won't crash because of an - // unhandled exception on the stream. - .on('error', () => {}); - try { - checksums = await FileChecksums.hashStream(entryStream, checksumBitmask); - } finally { - /** - * In the case the callback doesn't read the entire stream, {@link unzipper} will leave - * the file handle open. Drain the stream so the file handle can be released. The stream - * cannot be destroyed by the callback, or this will never resolve! - */ - await StreamPoly.autodrain(entryStream); + const entries: ArchiveEntry[] = []; + await new Promise((resolve, reject) => { + yauzl.open( + this.getFilePath(), + { lazyEntries: true }, + (zipError, zipFile) => { + if (zipError) { + reject(zipError); + return; } - } - const { crc32, ...checksumsWithoutCrc } = checksums; - - const archiveEntry = await ArchiveEntry.entryOf({ - archive: this, - entryPath: entryFile.path, - size: entryFile.uncompressedSize, - crc32: crc32 ?? entryFile.crc32.toString(16), - ...checksumsWithoutCrc, - }, checksumBitmask); - callback(undefined, archiveEntry); - }, - ); + + zipFile.readEntry(); + zipFile.on('entry', async (entryFile) => { + if (entryFile.fileName.endsWith('/')) { + zipFile.readEntry(); // continue + return; + } + + let checksums: ChecksumProps = {}; + if (checksumBitmask & ~ChecksumBitmask.CRC32) { + await new Promise((entryResolve) => { + zipFile.openReadStream(entryFile, async (entryError, entryStream) => { + if (entryError) { + reject(entryError); + return; + } + entryStream.on('error', reject); + + try { + checksums = await FileChecksums.hashStream(entryStream, checksumBitmask); + } catch (error) { + reject(error); + } finally { + entryStream.destroy(); + entryResolve(); + } + }); + }); + } + const { crc32, ...checksumsWithoutCrc } = checksums; + + try { + const archiveEntry = await ArchiveEntry.entryOf({ + archive: this, + entryPath: entryFile.fileName, + size: entryFile.uncompressedSize, + crc32: crc32 ?? entryFile.crc32.toString(16), + ...checksumsWithoutCrc, + }, checksumBitmask); + entries.push(archiveEntry); + } catch (error) { + reject(error); + } finally { + zipFile.readEntry(); // continue + } + }); + + zipFile.on('error', reject); + zipFile.on('close', () => resolve()); + }, + ); + }); + return entries; } async extractEntryToFile( @@ -96,33 +118,55 @@ export default class Zip extends Archive { return super.extractEntryToStream(entryPath, callback, start); } - const archive = await unzipper.Open.file(this.getFilePath()); + return new Promise((resolve, reject) => { + yauzl.open( + this.getFilePath(), + { lazyEntries: true }, + (zipError, zipFile) => { + if (zipError) { + reject(zipError); + return; + } - const entry = archive.files - .filter((entryFile) => entryFile.type === 'File') - .find((entryFile) => entryFile.path === entryPath.replace(/[\\/]/g, '/')); - if (!entry) { - // This should never happen, this likely means the zip file was modified after scanning - throw new Error(`didn't find entry '${entryPath}'`); - } + let result: T; + let foundEntry = false; + zipFile.readEntry(); + zipFile.on('entry', (entryFile) => { + if (entryFile.fileName !== entryPath.replace(/[\\/]/g, '/')) { + zipFile.readEntry(); // continue + return; + } - let stream: Entry; - try { - stream = entry.stream(); - } catch (error) { - throw new Error(`failed to read '${this.getFilePath()}|${entryPath}': ${error}`); - } + zipFile.openReadStream(entryFile, async (entryError, entryStream) => { + if (entryError) { + reject(new Error(`failed to read '${this.getFilePath()}|${entryPath}': ${entryError}`)); + return; + } + entryStream.on('error', reject); - try { - return await callback(stream); - } finally { - /** - * In the case the callback doesn't read the entire stream, {@link unzipper} will leave the - * file handle open. Drain the stream so the file handle can be released. The stream cannot - * be destroyed by the callback, or this will never resolve! - */ - await StreamPoly.autodrain(stream); - } + try { + result = await callback(entryStream); + foundEntry = true; + } catch (error) { + reject(error); + } finally { + entryStream.destroy(); + zipFile.readEntry(); // continue + } + }); + }); + + zipFile.on('error', reject); + zipFile.on('close', () => { + if (foundEntry) { + resolve(result); + } else { + reject(new Error(`didn't find entry '${entryPath}'`)); + } + }); + }, + ); + }); } async createArchive(inputToOutput: [File, ArchiveEntry][]): Promise {