From 2b3111f8a0da1c3bdc7979afee899aad76791d21 Mon Sep 17 00:00:00 2001 From: Christian Emmer <10749361+emmercm@users.noreply.github.com> Date: Sat, 29 Jun 2024 18:36:21 -0700 Subject: [PATCH] Feature: allow exact-matching of archives in DATs (#1175) --- docs/input/reading-archives.md | 16 +++++++++ src/igir.ts | 35 ++++++++++++++++--- src/modules/argumentsParser.ts | 12 ++++++- src/modules/movedRomDeleter.ts | 52 ++++++++++++++++++++-------- src/modules/romScanner.ts | 6 +++- src/modules/scanner.ts | 15 ++++++-- src/types/files/archives/zip.ts | 2 +- src/types/options.ts | 22 ++++++++++++ test/fixtures/dats/one.dat | 3 +- test/igir.test.ts | 19 +++++----- test/modules/argumentsParser.test.ts | 13 ++++++- test/modules/romScanner.test.ts | 25 ++++++++++--- 12 files changed, 179 insertions(+), 41 deletions(-) diff --git a/docs/input/reading-archives.md b/docs/input/reading-archives.md index 36efb6b7f..2be505334 100644 --- a/docs/input/reading-archives.md +++ b/docs/input/reading-archives.md @@ -30,6 +30,22 @@ Somewhat proprietary archive formats such as `.7z` and `.rar` require `igir` to This is why `igir` uses `.zip` as its output archive of choice, `.zip` files are easy and fast to read, even if they can't offer as high of compression as other formats. +## Exact archive matching + +Some DAT files such as the [libretro BIOS System.dat](https://github.com/libretro/libretro-database/blob/master/dat/System.dat) catalog archives such as zip files, rather than the contents of those archives. By default, `igir` will try to detect DATs like these and calculate checksums for all archive files, in addition to the files they contain. + +This adds a potentially non-trivial amount of processing time during ROM scanning, so this behavior can be turned off with the option: + +```text +--input-checksum-archives never +``` + +If for some reason `igir` isn't identifying an input file correctly as an archive, this additional processing can be forced with the option: + +```text +--input-checksum-archives always +``` + ## Checksum cache It can be expensive to calculate checksums of files within archives, especially MD5, SHA1, and SHA256. If `igir` needs to calculate a checksum that is not easily read from the archive (see above), it will cache the result in a file named `igir.cache`. This cached result will then be used as long as the input file's size and modified timestamp remain the same. diff --git a/src/igir.ts b/src/igir.ts index 259a4d7a4..50bcb1326 100644 --- a/src/igir.ts +++ b/src/igir.ts @@ -43,8 +43,9 @@ import DATStatus from './types/datStatus.js'; import File from './types/files/file.js'; import FileCache from './types/files/fileCache.js'; import { ChecksumBitmask } from './types/files/fileChecksums.js'; +import FileFactory from './types/files/fileFactory.js'; import IndexedFiles from './types/indexedFiles.js'; -import Options from './types/options.js'; +import Options, { InputChecksumArchivesMode } from './types/options.js'; import OutputFactory from './types/outputFactory.js'; import Patch from './types/patches/patch.js'; import ReleaseCandidate from './types/releaseCandidate.js'; @@ -100,7 +101,10 @@ export default class Igir { // Scan and process input files let dats = await this.processDATScanner(); - const indexedRoms = await this.processROMScanner(this.determineScanningBitmask(dats)); + const indexedRoms = await this.processROMScanner( + this.determineScanningBitmask(dats), + this.determineScanningChecksumArchives(dats), + ); const roms = indexedRoms.getFiles(); const patches = await this.processPatchScanner(); @@ -312,11 +316,34 @@ export default class Igir { return matchChecksum; } - private async processROMScanner(checksumBitmask: number): Promise { + private determineScanningChecksumArchives(dats: DAT[]): boolean { + if (this.options.getInputChecksumArchives() === InputChecksumArchivesMode.NEVER) { + return false; + } + if (this.options.getInputChecksumArchives() === InputChecksumArchivesMode.ALWAYS) { + return true; + } + return dats + .some((dat) => dat.getGames() + .some((game) => game.getRoms() + .some((rom) => { + const isArchive = FileFactory.isExtensionArchive(rom.getName()); + if (isArchive) { + this.logger.trace(`${dat.getNameShort()}: contains archives, enabling checksum calculation of raw archive contents`); + } + return isArchive; + }))); + } + + private async processROMScanner( + checksumBitmask: number, + checksumArchives: boolean, + ): Promise { const romScannerProgressBarName = 'Scanning for ROMs'; const romProgressBar = await this.logger.addProgressBar(romScannerProgressBarName); - const rawRomFiles = await new ROMScanner(this.options, romProgressBar).scan(checksumBitmask); + const rawRomFiles = await new ROMScanner(this.options, romProgressBar) + .scan(checksumBitmask, checksumArchives); await romProgressBar.setName('Detecting ROM headers'); const romFilesWithHeaders = await new ROMHeaderProcessor(this.options, romProgressBar) diff --git a/src/modules/argumentsParser.ts b/src/modules/argumentsParser.ts index cb1d105d3..6adfc0231 100644 --- a/src/modules/argumentsParser.ts +++ b/src/modules/argumentsParser.ts @@ -10,7 +10,7 @@ import ConsolePoly from '../polyfill/consolePoly.js'; import { ChecksumBitmask } from '../types/files/fileChecksums.js'; import ROMHeader from '../types/files/romHeader.js'; import Internationalization from '../types/internationalization.js'; -import Options, { GameSubdirMode, MergeMode } from '../types/options.js'; +import Options, { GameSubdirMode, InputChecksumArchivesMode, MergeMode } from '../types/options.js'; import PatchFactory from '../types/patches/patchFactory.js'; /** @@ -209,6 +209,16 @@ export default class ArgumentsParser { requiresArg: true, default: ChecksumBitmask[ChecksumBitmask.CRC32].toUpperCase(), }) + .option('input-checksum-archives', { + group: groupRomInput, + description: 'Calculate checksums of archive files themselves, allowing them to match files in DATs', + choices: Object.keys(InputChecksumArchivesMode) + .filter((mode) => Number.isNaN(Number(mode))) + .map((mode) => mode.toLowerCase()), + coerce: ArgumentsParser.getLastValue, // don't allow string[] values + requiresArg: true, + default: InputChecksumArchivesMode[InputChecksumArchivesMode.AUTO].toLowerCase(), + }) .option('dat', { group: groupDatInput, diff --git a/src/modules/movedRomDeleter.ts b/src/modules/movedRomDeleter.ts index bcc06b9db..27fc5fe80 100644 --- a/src/modules/movedRomDeleter.ts +++ b/src/modules/movedRomDeleter.ts @@ -2,7 +2,9 @@ import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js'; import ArrayPoly from '../polyfill/arrayPoly.js'; import fsPoly from '../polyfill/fsPoly.js'; import DAT from '../types/dats/dat.js'; +import Archive from '../types/files/archives/archive.js'; import ArchiveEntry from '../types/files/archives/archiveEntry.js'; +import ArchiveFile from '../types/files/archives/archiveFile.js'; import File from '../types/files/file.js'; import Module from './module.js'; @@ -79,21 +81,41 @@ export default class MovedROMDeleter extends Module { movedEntries.flatMap((file) => file.hashCode()), ); - const inputEntries = groupedInputRoms.get(filePath) ?? []; - - const unmovedEntries = inputEntries.filter((entry) => { - if (entry instanceof ArchiveEntry - && movedEntries.length === 1 - && !(movedEntries[0] instanceof ArchiveEntry) - && movedEntries[0].getFilePath() === entry.getFilePath() - ) { - // If the input archive entry was written as a raw archive, then consider it moved - return false; - } - - // Otherwise, the entry needs to have been explicitly moved - return !movedEntryHashCodes.has(entry.hashCode()); - }); + const inputFilesForPath = groupedInputRoms.get(filePath) ?? []; + const inputFileIsArchive = inputFilesForPath + .some((inputFile) => inputFile instanceof ArchiveEntry); + + const unmovedFiles = inputFilesForPath + .filter((inputFile) => !(inputFile instanceof ArchiveEntry)) + // The input archive entry needs to have been explicitly moved + .filter((inputFile) => !movedEntryHashCodes.has(inputFile.hashCode())); + + if (inputFileIsArchive && unmovedFiles.length === 0) { + // The input file is an archive, and it was fully extracted OR the archive file itself was + // an exact match and was moved as-is + return filePath; + } + + const unmovedArchiveEntries = inputFilesForPath + .filter(( + inputFile, + ): inputFile is ArchiveEntry => inputFile instanceof ArchiveEntry) + .filter((inputEntry) => { + if (movedEntries.length === 1 && movedEntries[0] instanceof ArchiveFile) { + // If the input archive was written as a raw archive, then consider it moved + return false; + } + + // Otherwise, the input archive entry needs to have been explicitly moved + return !movedEntryHashCodes.has(inputEntry.hashCode()); + }); + + if (inputFileIsArchive && unmovedArchiveEntries.length === 0) { + // The input file is an archive and it was fully zipped + return filePath; + } + + const unmovedEntries = [...unmovedFiles, ...unmovedArchiveEntries]; if (unmovedEntries.length > 0) { this.progressBar.logWarn(`${filePath}: not deleting moved file, ${unmovedEntries.length.toLocaleString()} archive entr${unmovedEntries.length !== 1 ? 'ies were' : 'y was'} unmatched:\n${unmovedEntries.sort().map((entry) => ` ${entry}`).join('\n')}`); return undefined; diff --git a/src/modules/romScanner.ts b/src/modules/romScanner.ts index 185d5ccc7..f84f9f46f 100644 --- a/src/modules/romScanner.ts +++ b/src/modules/romScanner.ts @@ -16,7 +16,10 @@ export default class ROMScanner extends Scanner { /** * Scan for ROM files. */ - async scan(checksumBitmask: number = ChecksumBitmask.CRC32): Promise { + async scan( + checksumBitmask: number = ChecksumBitmask.CRC32, + checksumArchives = false, + ): Promise { this.progressBar.logTrace('scanning ROM files'); await this.progressBar.setSymbol(ProgressBarSymbol.SEARCHING); await this.progressBar.reset(0); @@ -31,6 +34,7 @@ export default class ROMScanner extends Scanner { romFilePaths, this.options.getReaderThreads(), checksumBitmask, + checksumArchives, ); this.progressBar.logTrace('done scanning ROM files'); diff --git a/src/modules/scanner.ts b/src/modules/scanner.ts index 0b0355f3c..191a4a797 100644 --- a/src/modules/scanner.ts +++ b/src/modules/scanner.ts @@ -4,6 +4,7 @@ import ElasticSemaphore from '../elasticSemaphore.js'; import Defaults from '../globals/defaults.js'; import ArrayPoly from '../polyfill/arrayPoly.js'; import fsPoly from '../polyfill/fsPoly.js'; +import ArchiveEntry from '../types/files/archives/archiveEntry.js'; import File from '../types/files/file.js'; import FileFactory from '../types/files/fileFactory.js'; import Options from '../types/options.js'; @@ -30,6 +31,7 @@ export default abstract class Scanner extends Module { filePaths: string[], threads: number, checksumBitmask: number, + checksumArchives = false, ): Promise { return (await new DriveSemaphore(threads).map( filePaths, @@ -38,7 +40,7 @@ export default abstract class Scanner extends Module { const waitingMessage = `${inputFile} ...`; this.progressBar.addWaitingMessage(waitingMessage); - const files = await this.getFilesFromPath(inputFile, checksumBitmask); + const files = await this.getFilesFromPath(inputFile, checksumBitmask, checksumArchives); this.progressBar.removeWaitingMessage(waitingMessage); await this.progressBar.incrementDone(); @@ -60,6 +62,7 @@ export default abstract class Scanner extends Module { private async getFilesFromPath( filePath: string, checksumBitmask: number, + checksumArchives = false, ): Promise { try { const totalKilobytes = await fsPoly.size(filePath) / 1024; @@ -72,7 +75,15 @@ export default abstract class Scanner extends Module { return []; } } - return FileFactory.filesFrom(filePath, checksumBitmask); + + const filesFromPath = await FileFactory.filesFrom(filePath, checksumBitmask); + + const fileIsArchive = filesFromPath.some((file) => file instanceof ArchiveEntry); + if (checksumArchives && fileIsArchive) { + filesFromPath.push(await FileFactory.fileFrom(filePath, checksumBitmask)); + } + + return filesFromPath; }, totalKilobytes, ); diff --git a/src/types/files/archives/zip.ts b/src/types/files/archives/zip.ts index 43a4da5d8..a1662ed0f 100644 --- a/src/types/files/archives/zip.ts +++ b/src/types/files/archives/zip.ts @@ -22,7 +22,7 @@ export default class Zip extends Archive { } static getExtensions(): string[] { - return ['.zip']; + return ['.zip', '.apk', '.ipa', '.jar', '.pk3']; } // eslint-disable-next-line class-methods-use-this diff --git a/src/types/options.ts b/src/types/options.ts index 332edcb73..53fa5b711 100644 --- a/src/types/options.ts +++ b/src/types/options.ts @@ -22,6 +22,15 @@ import DAT from './dats/dat.js'; import File from './files/file.js'; import { ChecksumBitmask } from './files/fileChecksums.js'; +export enum InputChecksumArchivesMode { + // Never calculate the checksum of archive files + NEVER = 1, + // Calculate the checksum of archive files if DATs reference archives + AUTO = 2, + // Always calculate the checksum of archive files + ALWAYS = 3, +} + export enum MergeMode { // Clones contain all parent ROMs, all games contain BIOS & device ROMs FULLNONMERGED = 1, @@ -49,6 +58,7 @@ export interface OptionsProps { readonly input?: string[], readonly inputExclude?: string[], readonly inputMinChecksum?: string, + readonly inputChecksumArchives?: string, readonly dat?: string[], readonly datExclude?: string[], @@ -166,6 +176,8 @@ export default class Options implements OptionsProps { readonly inputMinChecksum?: string; + readonly inputChecksumArchives?: string; + readonly dat: string[]; readonly datExclude: string[]; @@ -355,6 +367,7 @@ export default class Options implements OptionsProps { this.input = options?.input ?? []; this.inputExclude = options?.inputExclude ?? []; this.inputMinChecksum = options?.inputMinChecksum; + this.inputChecksumArchives = options?.inputChecksumArchives; this.dat = options?.dat ?? []; this.datExclude = options?.datExclude ?? []; @@ -751,6 +764,15 @@ export default class Options implements OptionsProps { return ChecksumBitmask[checksumBitmask as keyof typeof ChecksumBitmask]; } + getInputChecksumArchives(): InputChecksumArchivesMode | undefined { + const checksumMode = Object.keys(InputChecksumArchivesMode) + .find((mode) => mode.toLowerCase() === this.inputChecksumArchives?.toLowerCase()); + if (!checksumMode) { + return undefined; + } + return InputChecksumArchivesMode[checksumMode as keyof typeof InputChecksumArchivesMode]; + } + /** * Were any DAT paths provided? */ diff --git a/test/fixtures/dats/one.dat b/test/fixtures/dats/one.dat index 0fb42a403..beb7461a8 100644 --- a/test/fixtures/dats/one.dat +++ b/test/fixtures/dats/one.dat @@ -29,7 +29,8 @@ Lorem Ipsum - + + One Three diff --git a/test/igir.test.ts b/test/igir.test.ts index b0485f662..356c397e5 100644 --- a/test/igir.test.ts +++ b/test/igir.test.ts @@ -178,7 +178,7 @@ describe('with explicit DATs', () => { [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'], [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [`${path.join('One', 'One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('One', 'One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('One', 'One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -222,7 +222,7 @@ describe('with explicit DATs', () => { expect(result.outputFilesAndCrcs).toEqual([ // Fizzbuzz.nes is explicitly missing! ['Foobar.lnx', 'b22c9747'], - ['Lorem Ipsum.rom', '70856527'], + ['Lorem Ipsum.zip|loremipsum.rom', '70856527'], [`${path.join('One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -278,7 +278,6 @@ describe('with explicit DATs', () => { [path.join('nes', 'smdb', 'Hardware Target Game Database', 'Dummy', 'Fizzbuzz.nes'), '370517b5'], ['one.rom', '00000000'], // explicitly not deleted, it is not in an extension subdirectory [`${path.join('rar', 'Headered', 'LCDTestROM.lnx.rar')}|LCDTestROM.lnx`, '2d251538'], - [path.join('rom', 'One', 'Lorem Ipsum.rom'), '70856527'], [path.join('rom', 'One', 'Three Four Five', 'Five.rom'), '3e5daf67'], [path.join('rom', 'One', 'Three Four Five', 'Four.rom'), '1cf3ca74'], [path.join('rom', 'One', 'Three Four Five', 'Three.rom'), 'ff46c5d8'], @@ -296,6 +295,7 @@ describe('with explicit DATs', () => { [path.join('rom', 'smdb', 'Hardware Target Game Database', 'Patchable', 'C01173E.rom'), 'dfaebe28'], [path.join('smc', 'Headered', 'speed_test_v51.smc'), '9adca6cc'], [`${path.join('zip', 'Headered', 'fds_joypad_test.fds.zip')}|fds_joypad_test.fds`, '1e58456d'], + [`${path.join('zip', 'One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('zip', 'One', 'One Three.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -332,7 +332,6 @@ describe('with explicit DATs', () => { expect(result.outputFilesAndCrcs).toEqual([ [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], [path.join('One', 'One Three', 'One.rom'), 'f817a89f'], [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'], @@ -371,7 +370,6 @@ describe('with explicit DATs', () => { expect(result.outputFilesAndCrcs).toEqual([ [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], [path.join('One', 'One Three', 'One.rom'), 'f817a89f'], [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'], @@ -425,7 +423,7 @@ describe('with explicit DATs', () => { [path.join('igir combined', 'KDULVQN.rom'), 'b1c303e4'], [path.join('igir combined', 'LCDTestROM.lnx'), '2d251538'], [path.join('igir combined', 'LCDTestROM.lyx'), '42583855'], - [path.join('igir combined', 'Lorem Ipsum.rom'), '70856527'], + [`${path.join('igir combined', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [path.join('igir combined', 'One Three', 'One.rom'), 'f817a89f'], [path.join('igir combined', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('igir combined', 'speed_test_v51.sfc'), '8beffd94'], @@ -459,6 +457,7 @@ describe('with explicit DATs', () => { path.join('raw', 'loremipsum.rom'), path.join('raw', 'one.rom'), path.join('raw', 'three.rom'), + path.join('zip', 'loremipsum.zip'), ]); expect(result.cleanedFiles).toHaveLength(0); }); @@ -560,7 +559,7 @@ describe('with explicit DATs', () => { [`${path.join('Headerless', 'speed_test_v51.zip')}|speed_test_v51.sfc`, '8beffd94'], [`${path.join('One', 'Fizzbuzz.zip')}|Fizzbuzz.nes`, '370517b5'], [`${path.join('One', 'Foobar.zip')}|Foobar.lnx`, 'b22c9747'], - [`${path.join('One', 'Lorem Ipsum.zip')}|Lorem Ipsum.rom`, '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|Lorem Ipsum.zip`, '7ee77289'], [`${path.join('One', 'One Three.zip')}|One.rom`, 'f817a89f'], [`${path.join('One', 'One Three.zip')}|Three.rom`, 'ff46c5d8'], [`${path.join('One', 'Three Four Five.zip')}|Five.rom`, '3e5daf67'], @@ -619,7 +618,7 @@ describe('with explicit DATs', () => { ['Headerless.zip|speed_test_v51.sfc', '8beffd94'], ['One.zip|Fizzbuzz.nes', '370517b5'], ['One.zip|Foobar.lnx', 'b22c9747'], - ['One.zip|Lorem Ipsum.rom', '70856527'], + ['One.zip|Lorem Ipsum.zip', '7ee77289'], [`One.zip|${path.join('One Three', 'One.rom')}`, 'f817a89f'], [`One.zip|${path.join('One Three', 'Three.rom')}`, 'ff46c5d8'], [`One.zip|${path.join('Three Four Five', 'Five.rom')}`, '3e5daf67'], @@ -664,7 +663,7 @@ describe('with explicit DATs', () => { [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc -> ${path.join('', 'headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'], [`${path.join('One', 'Fizzbuzz.nes')} -> ${path.join('', 'raw', 'fizzbuzz.nes')}`, '370517b5'], [`${path.join('One', 'Foobar.lnx')} -> ${path.join('', 'foobar.lnx')}`, 'b22c9747'], - [`${path.join('One', 'Lorem Ipsum.rom')} -> ${path.join('', 'raw', 'loremipsum.rom')}`, '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom -> ${path.join('', 'zip', 'loremipsum.zip')}|loremipsum.rom`, '70856527'], [`${path.join('One', 'One Three.zip')}|${path.join('1', 'one.rom')} -> ${path.join('', 'zip', 'onetwothree.zip')}|${path.join('1', 'one.rom')}`, 'f817a89f'], [`${path.join('One', 'One Three.zip')}|${path.join('2', 'two.rom')} -> ${path.join('', 'zip', 'onetwothree.zip')}|${path.join('2', 'two.rom')}`, '96170874'], [`${path.join('One', 'One Three.zip')}|${path.join('3', 'three.rom')} -> ${path.join('', 'zip', 'onetwothree.zip')}|${path.join('3', 'three.rom')}`, 'ff46c5d8'], @@ -721,7 +720,7 @@ describe('with explicit DATs', () => { [path.join('Headerless', 'speed_test_v51.sfc'), '8beffd94'], [path.join('One', 'Fizzbuzz.nes'), '370517b5'], [path.join('One', 'Foobar.lnx'), 'b22c9747'], - [path.join('One', 'Lorem Ipsum.rom'), '70856527'], + [`${path.join('One', 'Lorem Ipsum.zip')}|loremipsum.rom`, '70856527'], [path.join('One', 'One Three', 'One.rom'), 'f817a89f'], [path.join('One', 'One Three', 'Three.rom'), 'ff46c5d8'], [path.join('One', 'Three Four Five', 'Five.rom'), '3e5daf67'], diff --git a/test/modules/argumentsParser.test.ts b/test/modules/argumentsParser.test.ts index 90c525602..81edc637d 100644 --- a/test/modules/argumentsParser.test.ts +++ b/test/modules/argumentsParser.test.ts @@ -9,7 +9,7 @@ import FsPoly from '../../src/polyfill/fsPoly.js'; import Header from '../../src/types/dats/logiqx/header.js'; import LogiqxDAT from '../../src/types/dats/logiqx/logiqxDat.js'; import { ChecksumBitmask } from '../../src/types/files/fileChecksums.js'; -import { GameSubdirMode, MergeMode } from '../../src/types/options.js'; +import { GameSubdirMode, InputChecksumArchivesMode, MergeMode } from '../../src/types/options.js'; const dummyRequiredArgs = ['--input', os.devNull, '--output', os.devNull]; const dummyCommandAndRequiredArgs = ['copy', ...dummyRequiredArgs]; @@ -106,6 +106,7 @@ describe('options', () => { expect(options.getInputPaths()).toEqual([os.devNull]); expect(options.getInputMinChecksum()).toEqual(ChecksumBitmask.CRC32); + expect(options.getInputChecksumArchives()).toEqual(InputChecksumArchivesMode.AUTO); expect(options.getDatNameRegex()).toBeUndefined(); expect(options.getDatNameRegexExclude()).toBeUndefined(); @@ -239,6 +240,16 @@ describe('options', () => { expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--input-min-checksum', 'SHA256', '--input-min-checksum', 'CRC32']).getInputMinChecksum()).toEqual(ChecksumBitmask.CRC32); }); + it('should parse "input-checksum-archives"', () => { + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs]).getInputChecksumArchives()) + .toEqual(InputChecksumArchivesMode.AUTO); + expect(() => argumentsParser.parse([...dummyCommandAndRequiredArgs, '--input-checksum-archives', 'foobar']).getInputChecksumArchives()).toThrow(/invalid values/i); + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--input-checksum-archives', 'never']).getInputChecksumArchives()).toEqual(InputChecksumArchivesMode.NEVER); + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--input-checksum-archives', 'auto']).getInputChecksumArchives()).toEqual(InputChecksumArchivesMode.AUTO); + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--input-checksum-archives', 'always']).getInputChecksumArchives()).toEqual(InputChecksumArchivesMode.ALWAYS); + expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--input-checksum-archives', 'always', '--input-checksum-archives', 'never']).getInputChecksumArchives()).toEqual(InputChecksumArchivesMode.NEVER); + }); + it('should parse "dat"', async () => { expect(() => argumentsParser.parse(['report', '--input', os.devNull])).toThrow(/missing required argument/i); expect(() => argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat'])).toThrow(/not enough arguments/i); diff --git a/test/modules/romScanner.test.ts b/test/modules/romScanner.test.ts index b259b68cc..bfa2ccb35 100644 --- a/test/modules/romScanner.test.ts +++ b/test/modules/romScanner.test.ts @@ -4,7 +4,8 @@ import path from 'node:path'; import Temp from '../../src/globals/temp.js'; import ROMScanner from '../../src/modules/romScanner.js'; import fsPoly from '../../src/polyfill/fsPoly.js'; -import Options from '../../src/types/options.js'; +import { ChecksumBitmask } from '../../src/types/files/fileChecksums.js'; +import Options, { OptionsProps } from '../../src/types/options.js'; import ProgressBarFake from '../console/progressBarFake.js'; function createRomScanner(input: string[], inputExclude: string[] = []): ROMScanner { @@ -40,6 +41,18 @@ describe('multiple files', () => { await expect(createRomScanner(['test/fixtures/roms/**/*', 'test/fixtures/roms/**/*.{rom,zip}']).scan()).resolves.toHaveLength(expectedRomFiles); }); + test.each([ + [{ input: ['test/fixtures/roms'] }, 90], + [{ input: ['test/fixtures/roms/7z'] }, 12], + [{ input: ['test/fixtures/roms/rar'] }, 12], + [{ input: ['test/fixtures/roms/tar'] }, 12], + [{ input: ['test/fixtures/roms/zip'] }, 15], + ] satisfies [OptionsProps, number][])('should calculate checksums of archives: %s', async (optionsProps, expectedRomFiles) => { + const scannedFiles = await new ROMScanner(new Options(optionsProps), new ProgressBarFake()) + .scan(ChecksumBitmask.CRC32, true); + expect(scannedFiles).toHaveLength(expectedRomFiles); + }); + it('should scan multiple files with some file exclusions', async () => { await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(44); await expect(createRomScanner(['test/fixtures/roms/**/*'], ['test/fixtures/roms/**/*.rom', 'test/fixtures/roms/**/*.rom']).scan()).resolves.toHaveLength(44); @@ -169,8 +182,10 @@ describe('multiple files', () => { }); }); -it('should scan single files', async () => { - await expect(createRomScanner(['test/fixtures/roms/empty.*']).scan()).resolves.toHaveLength(1); - await expect(createRomScanner(['test/fixtures/*/empty.rom']).scan()).resolves.toHaveLength(1); - await expect(createRomScanner(['test/fixtures/roms/empty.rom']).scan()).resolves.toHaveLength(1); +describe('single files', () => { + it('should scan single files with no exclusions', async () => { + await expect(createRomScanner(['test/fixtures/roms/empty.*']).scan()).resolves.toHaveLength(1); + await expect(createRomScanner(['test/fixtures/*/empty.rom']).scan()).resolves.toHaveLength(1); + await expect(createRomScanner(['test/fixtures/roms/empty.rom']).scan()).resolves.toHaveLength(1); + }); });