Feature: temp dir option (#1041)

docs/advanced/temp-dir.md

@@ -0,0 +1,28 @@
+# Temp Directory
+
+`igir` needs to write some temporary files to disk for a few reasons:
+
+- Downloading [DAT URLs](../dats/processing.md#scanning-for-dats) to disk before parsing
+- Extracting [some archives](../input/reading-archives.md) to disk during scanning, and when reading when extracting or [zipping](../output/writing-archives.md)
+
+Temporary files are ones that are deleted as soon as `igir` no longer needs them for processing. `igir` will also delete any leftover temporary files on exit.
+
+`igir` will use your operating system's temporary directory for these files by default.  The option `--temp-dir <path>` is provided to let you change the directory, and you may want to do this for a few reasons:
+
+- Your operating system drive has minimal space available
+- You want to protect your operating system drive from excess wear and tear
+- You want to use a "RAM disk" instead of a real drive
+
+## RAM disks
+
+### :simple-windowsxp: Windows
+
+There are no tools built-in to Windows that can create a RAM disk. The open source [ImDisk Toolkit](https://sourceforge.net/projects/imdisk-toolkit/) is a popular option.
+
+### :simple-apple: macOS
+
+The built-in `diskutil` and `hdiutil` tools can be used to create and mount a RAM disk. Alex T has some instructions in a [GitHub gist](https://gist.github.com/htr3n/344f06ba2bb20b1056d7d5570fe7f596).
+
+### :simple-linux: Linux
+
+`tmpfs` is a tool that comes with most Linux distributions that is used for creating RAM disks. Oracle has [a guide](https://docs.oracle.com/cd/E18752_01/html/817-5093/fscreate-99040.html) on the tool.

mkdocs.yml

@@ -100,6 +100,7 @@ nav:
           - output/cleaning.md
       - Advanced:
           - advanced/logging.md
+          - advanced/temp-dir.md
           - advanced/troubleshooting.md
           - advanced/internals.md
       - Misc:

src/globals/defaults.ts

@@ -1,24 +1,9 @@
-import os from 'node:os';
-import path from 'node:path';
-
 import fsPoly from '../polyfill/fsPoly.js';
-import Package from './package.js';
-
-const GLOBAL_TEMP_DIR = fsPoly.mkdtempSync(path.join(os.tmpdir(), Package.NAME));
-process.once('beforeExit', async () => {
-  // WARN: Jest won't call this: https://github.com/jestjs/jest/issues/10927
-  await fsPoly.rm(GLOBAL_TEMP_DIR, {
-    force: true,
-    recursive: true,
-  });
-});
 
 /**
  * A static class of globals that are determined at startup, to be used widely.
  */
 export default class Defaults {
-  static readonly GLOBAL_TEMP_DIR = GLOBAL_TEMP_DIR;
-
   /**
    * A reasonable max of filesystem threads for operations such as:
    * @example

src/globals/temp.ts

@@ -0,0 +1,32 @@
+import os from 'node:os';
+import path from 'node:path';
+
+import moment from 'moment';
+
+import FsPoly from '../polyfill/fsPoly.js';
+import Package from './package.js';
+
+/**
+ * A static class of constants for temp directories, to be used widely.
+ */
+export default class Temp {
+  // Note: this default path is explicitly not created immediately in case it gets changed by CLI
+  // options
+  private static globalTempDir = path.join(os.tmpdir(), Package.NAME, moment().format('YYYYMMDD-HHmmss'));
+
+  public static getTempDir(): string {
+    return this.globalTempDir;
+  }
+
+  public static setTempDir(globalTempDir: string): void {
+    this.globalTempDir = globalTempDir;
+  }
+}
+
+process.once('beforeExit', async () => {
+  // WARN: Jest won't call this: https://github.com/jestjs/jest/issues/10927
+  await FsPoly.rm(Temp.getTempDir(), {
+    force: true,
+    recursive: true,
+  });
+});

src/igir.ts

@@ -8,8 +8,8 @@ import isAdmin from 'is-admin';
 import Logger from './console/logger.js';
 import ProgressBar, { ProgressBarSymbol } from './console/progressBar.js';
 import ProgressBarCLI from './console/progressBarCli.js';
-import Defaults from './globals/defaults.js';
 import Package from './globals/package.js';
+import Temp from './globals/temp.js';
 import CandidateArchiveFileHasher from './modules/candidateArchiveFileHasher.js';
 import CandidateCombiner from './modules/candidateCombiner.js';
 import CandidateGenerator from './modules/candidateGenerator.js';
@@ -66,14 +66,16 @@ export default class Igir {
    * The main method for this application.
    */
   async main(): Promise<void> {
+    Temp.setTempDir(this.options.getTempDir());
+
     // Windows 10 may require admin privileges to symlink at all
     // @see https://github.com/nodejs/node/issues/18518
     if (this.options.shouldLink()
       && this.options.getSymlink()
       && process.platform === 'win32'
     ) {
       this.logger.trace('checking Windows for symlink permissions');
-      if (!await FsPoly.canSymlink(Defaults.GLOBAL_TEMP_DIR)) {
+      if (!await FsPoly.canSymlink(Temp.getTempDir())) {
         if (!await isAdmin()) {
           throw new Error(`${Package.NAME} does not have permissions to create symlinks, please try running as administrator`);
         }

src/modules/argumentsParser.ts

@@ -810,6 +810,13 @@ export default class ArgumentsParser {
         requiresArg: true,
         default: Defaults.ROM_WRITER_ADDITIONAL_RETRIES,
       })
+      .options('temp-dir', {
+        group: groupHelpDebug,
+        description: 'Path to a directory for temporary files',
+        type: 'string',
+        coerce: ArgumentsParser.getLastValue, // don't allow string[] values
+        requiresArg: true,
+      })
       .option('disable-cache', {
         group: groupHelpDebug,
         description: 'Disable the file checksum cache',

src/modules/reportGenerator.ts

@@ -1,3 +1,5 @@
+import path from 'node:path';
+
 import ProgressBar from '../console/progressBar.js';
 import FsPoly from '../polyfill/fsPoly.js';
 import DATStatus, { GameStatus } from '../types/datStatus.js';
@@ -71,6 +73,10 @@ export default class ReportGenerator extends Module {
     const cleanedCsv = await DATStatus.filesToCsv(cleanedOutputFiles, GameStatus.DELETED);
 
     this.progressBar.logInfo(`writing report '${reportPath}'`);
+    const reportPathDir = path.dirname(reportPath);
+    if (!await FsPoly.exists(reportPathDir)) {
+      await FsPoly.mkdir(reportPathDir, { recursive: true });
+    }
     const rows = [
       ...matchedFileCsvs,
       duplicateCsv,

src/polyfill/fsPoly.ts

@@ -214,25 +214,6 @@ export default class FsPoly {
     }
   }
 
-  /**
-   * mkdtempSync() takes a path "prefix" that's concatenated with random characters. Ignore that
-   * behavior and instead assume we always want to specify a root temp directory.
-   */
-  static mkdtempSync(rootDir: string): string {
-    const rootDirProcessed = rootDir.replace(/[\\/]+$/, '') + path.sep;
-
-    try {
-      fs.mkdirSync(rootDirProcessed, { recursive: true });
-
-      return fs.mkdtempSync(rootDirProcessed);
-    } catch {
-      const backupDir = path.join(process.cwd(), 'tmp') + path.sep;
-      fs.mkdirSync(backupDir, { recursive: true });
-
-      return fs.mkdtempSync(backupDir);
-    }
-  }
-
   static async mktemp(prefix: string): Promise<string> {
     for (let i = 0; i < 10; i += 1) {
       const randomExtension = crypto.randomBytes(4).readUInt32LE().toString(36);

src/types/files/archives/archive.ts

@@ -1,7 +1,7 @@
 import path from 'node:path';
 import { Readable } from 'node:stream';
 
-import Defaults from '../../../globals/defaults.js';
+import Temp from '../../../globals/temp.js';
 import fsPoly from '../../../polyfill/fsPoly.js';
 import File from '../file.js';
 import ArchiveEntry from './archiveEntry.js';
@@ -33,7 +33,7 @@ export default abstract class Archive {
     callback: (tempFile: string) => (T | Promise<T>),
   ): Promise<T> {
     const tempFile = await fsPoly.mktemp(path.join(
-      Defaults.GLOBAL_TEMP_DIR,
+      Temp.getTempDir(),
       path.basename(entryPath),
     ));
 

src/types/files/archives/sevenZip.ts

@@ -5,6 +5,7 @@ import async, { AsyncResultCallback } from 'async';
 import { Mutex } from 'async-mutex';
 
 import Defaults from '../../../globals/defaults.js';
+import Temp from '../../../globals/temp.js';
 import fsPoly from '../../../polyfill/fsPoly.js';
 import Archive from './archive.js';
 import ArchiveEntry from './archiveEntry.js';
@@ -95,7 +96,7 @@ export default class SevenZip extends Archive {
     entryPath: string,
     extractedFilePath: string,
   ): Promise<void> {
-    const tempDir = await fsPoly.mkdtemp(path.join(Defaults.GLOBAL_TEMP_DIR, '7z'));
+    const tempDir = await fsPoly.mkdtemp(path.join(Temp.getTempDir(), '7z'));
     try {
       let tempFile = path.join(tempDir, entryPath);
       await new Promise<void>((resolve, reject) => {

src/types/files/file.ts

@@ -8,6 +8,7 @@ import {
 } from 'class-transformer';
 
 import Defaults from '../../globals/defaults.js';
+import Temp from '../../globals/temp.js';
 import FilePoly from '../../polyfill/filePoly.js';
 import fsPoly from '../../polyfill/fsPoly.js';
 import URLPoly from '../../polyfill/urlPoly.js';
@@ -267,9 +268,13 @@ export default class File implements FileProps {
     callback: (tempFile: string) => (T | Promise<T>),
   ): Promise<T> {
     const tempFile = await fsPoly.mktemp(path.join(
-      Defaults.GLOBAL_TEMP_DIR,
+      Temp.getTempDir(),
       path.basename(this.getFilePath()),
     ));
+    const tempDir = path.dirname(tempFile);
+    if (!await fsPoly.exists(tempDir)) {
+      await fsPoly.mkdir(tempDir, { recursive: true });
+    }
     await fsPoly.copyFile(this.getFilePath(), tempFile);
 
     try {
@@ -309,7 +314,7 @@ export default class File implements FileProps {
 
     // Complex case: create a temp file with the header removed
     const tempFile = await fsPoly.mktemp(path.join(
-      Defaults.GLOBAL_TEMP_DIR,
+      Temp.getTempDir(),
       path.basename(this.getExtractedFilePath()),
     ));
     if (patch) {
@@ -359,7 +364,7 @@ export default class File implements FileProps {
 
     // Complex case: create a temp patched file and then create read stream at an offset
     const tempFile = await fsPoly.mktemp(path.join(
-      Defaults.GLOBAL_TEMP_DIR,
+      Temp.getTempDir(),
       path.basename(this.getExtractedFilePath()),
     ));
     try {
@@ -414,7 +419,7 @@ export default class File implements FileProps {
       return this;
     }
 
-    const filePath = await fsPoly.mktemp(path.join(Defaults.GLOBAL_TEMP_DIR, tempPrefix));
+    const filePath = await fsPoly.mktemp(path.join(Temp.getTempDir(), tempPrefix));
     return this.downloadToPath(filePath);
   }
 

src/types/options.ts

@@ -14,6 +14,7 @@ import moment from 'moment';
 
 import LogLevel from '../console/logLevel.js';
 import Defaults from '../globals/defaults.js';
+import Temp from '../globals/temp.js';
 import ArrayPoly from '../polyfill/arrayPoly.js';
 import fsPoly, { FsWalkCallback } from '../polyfill/fsPoly.js';
 import URLPoly from '../polyfill/urlPoly.js';
@@ -143,6 +144,7 @@ export interface OptionsProps {
   readonly readerThreads?: number,
   readonly writerThreads?: number,
   readonly writeRetry?: number,
+  readonly tempDir?: string,
   readonly disableCache?: boolean,
   readonly cachePath?: string,
   readonly verbose?: number,
@@ -336,6 +338,8 @@ export default class Options implements OptionsProps {
 
   readonly writeRetry: number;
 
+  readonly tempDir: string;
+
   readonly disableCache: boolean;
 
   readonly cachePath?: string;
@@ -446,6 +450,7 @@ export default class Options implements OptionsProps {
     this.readerThreads = Math.max(options?.readerThreads ?? 0, 1);
     this.writerThreads = Math.max(options?.writerThreads ?? 0, 1);
     this.writeRetry = Math.max(options?.writeRetry ?? 0, 0);
+    this.tempDir = options?.tempDir ?? Temp.getTempDir();
     this.disableCache = options?.disableCache ?? false;
     this.cachePath = options?.cachePath;
     this.verbose = options?.verbose ?? 0;
@@ -818,7 +823,7 @@ export default class Options implements OptionsProps {
   }
 
   getOutput(): string {
-    return this.shouldWrite() ? this.output : Defaults.GLOBAL_TEMP_DIR;
+    return this.shouldWrite() ? this.output : this.getTempDir();
   }
 
   /**
@@ -1202,6 +1207,10 @@ export default class Options implements OptionsProps {
     return this.writeRetry;
   }
 
+  getTempDir(): string {
+    return this.tempDir;
+  }
+
   getDisableCache(): boolean {
     return this.disableCache;
   }