Add support for EXTENSIONS_LOCATION setting (directus#20207)

Co-authored-by: ian <[email protected]>
Co-authored-by: Brainslug <[email protected]>
Co-authored-by: Pascal Jufer <[email protected]>

.changeset/few-mice-know.md

@@ -0,0 +1,5 @@
+---
+'@directus/api': minor
+---
+
+Added configuration to allow extensions to be managed in a configured storage location

.eslintignore

@@ -9,7 +9,9 @@ dist/
 *.yml
 *.yaml
 *.md
+*.txt
 *.json
 *.scss
 *.css
 *.svg
+Dockerfile

Dockerfile

@@ -54,8 +54,6 @@ EXPOSE 8055
 ENV \
 	DB_CLIENT="sqlite3" \
 	DB_FILENAME="/directus/database/database.sqlite" \
-	EXTENSIONS_PATH="/directus/extensions" \
-	STORAGE_LOCAL_ROOT="/directus/uploads" \
 	NODE_ENV="production" \
 	NPM_CONFIG_UPDATE_NOTIFIER="false"
 

api/package.json

@@ -145,6 +145,7 @@
 		"openid-client": "5.4.2",
 		"ora": "6.3.1",
 		"otplib": "12.0.1",
+		"p-queue": "7.4.1",
 		"papaparse": "5.4.1",
 		"pino": "8.14.1",
 		"pino-http": "8.3.3",

api/src/database/index.ts

@@ -10,6 +10,7 @@ import path from 'path';
 import { performance } from 'perf_hooks';
 import { promisify } from 'util';
 import env from '../env.js';
+import { getExtensionsPath } from '../extensions/lib/get-extensions-path.js';
 import logger from '../logger.js';
 import type { DatabaseClient } from '../types/index.js';
 import { getConfigFromEnv } from '../utils/get-config-from-env.js';
@@ -258,7 +259,7 @@ export async function validateMigrations(): Promise<boolean> {
 	try {
 		let migrationFiles = await fse.readdir(path.join(__dirname, 'migrations'));
 
-		const customMigrationsPath = path.resolve(env['EXTENSIONS_PATH'], 'migrations');
+		const customMigrationsPath = path.resolve(getExtensionsPath(), 'migrations');
 
 		let customMigrationFiles =
 			((await fse.pathExists(customMigrationsPath)) && (await fse.readdir(customMigrationsPath))) || [];

api/src/database/migrations/run.ts

@@ -6,7 +6,7 @@ import { dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import path from 'path';
 import { flushCaches } from '../../cache.js';
-import env from '../../env.js';
+import { getExtensionsPath } from '../../extensions/lib/get-extensions-path.js';
 import logger from '../../logger.js';
 import type { Migration } from '../../types/index.js';
 import getModuleDefault from '../../utils/get-module-default.js';
@@ -16,7 +16,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 export default async function run(database: Knex, direction: 'up' | 'down' | 'latest', log = true): Promise<void> {
 	let migrationFiles = await fse.readdir(__dirname);
 
-	const customMigrationsPath = path.resolve(env['EXTENSIONS_PATH'], 'migrations');
+	const customMigrationsPath = path.resolve(getExtensionsPath(), 'migrations');
 
 	let customMigrationFiles =
 		((await fse.pathExists(customMigrationsPath)) && (await fse.readdir(customMigrationsPath))) || [];

api/src/env.ts

@@ -3,8 +3,8 @@
  * For all possible keys, see: https://docs.directus.io/self-hosted/config-options/
  */
 
-import { parseJSON, toArray } from '@directus/utils';
 import { JAVASCRIPT_FILE_EXTS } from '@directus/constants';
+import { parseJSON, toArray } from '@directus/utils';
 import dotenv from 'dotenv';
 import fs from 'fs';
 import { clone, toNumber, toString } from 'lodash-es';
@@ -35,6 +35,7 @@ const allowedEnvironmentVars = [
 	'QUERY_LIMIT_MAX',
 	'QUERY_LIMIT_DEFAULT',
 	'ROBOTS_TXT',
+	'TEMP_PATH',
 	// server
 	'SERVER_.+',
 	// database
@@ -163,6 +164,7 @@ const allowedEnvironmentVars = [
 	'AUTH_.+_SP.+',
 	// extensions
 	'PACKAGE_FILE_LOCATION',
+	'EXTENSIONS_LOCATION',
 	'EXTENSIONS_PATH',
 	'EXTENSIONS_AUTO_RELOAD',
 	'EXTENSIONS_CACHE_TTL',
@@ -225,6 +227,8 @@ export const defaults: Record<string, any> = {
 	MAX_BATCH_MUTATION: Infinity,
 	ROBOTS_TXT: 'User-agent: *\nDisallow: /',
 
+	TEMP_PATH: './node_modules/.directus',
+
 	DB_EXCLUDE_TABLES: 'spatial_ref_sys,sysdiagrams',
 
 	STORAGE_LOCATIONS: 'local',

api/src/extensions/lib/get-extensions-path.ts

@@ -0,0 +1,10 @@
+import { join } from 'path';
+import env from '../../env.js';
+
+export const getExtensionsPath = () => {
+	if (env['EXTENSIONS_LOCATION']) {
+		return join(env['TEMP_PATH'], 'extensions');
+	}
+
+	return env['EXTENSIONS_PATH'];
+};

api/src/extensions/lib/get-extensions.ts

@@ -1,15 +1,16 @@
 import type { Extension } from '@directus/extensions';
 import { getLocalExtensions, getPackageExtensions, resolvePackageExtensions } from '@directus/extensions/node';
 import env from '../../env.js';
+import { getExtensionsPath } from './get-extensions-path.js';
 
 export const getExtensions = async () => {
-	const localExtensions = await getLocalExtensions(env['EXTENSIONS_PATH']);
+	const localExtensions = await getLocalExtensions(getExtensionsPath());
 
 	const loadedNames = localExtensions.map(({ name }) => name);
 
 	const filterDuplicates = ({ name }: Extension) => loadedNames.includes(name) === false;
 
-	const localPackageExtensions = (await resolvePackageExtensions(env['EXTENSIONS_PATH'])).filter((extension) =>
+	const localPackageExtensions = (await resolvePackageExtensions(getExtensionsPath())).filter((extension) =>
 		filterDuplicates(extension)
 	);
 

api/src/extensions/lib/sync-extensions.ts

@@ -0,0 +1,82 @@
+import { NESTED_EXTENSION_TYPES } from '@directus/extensions';
+import { ensureExtensionDirs } from '@directus/extensions/node';
+import mid from 'node-machine-id';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import Queue from 'p-queue';
+import env from '../../env.js';
+import logger from '../../logger.js';
+import { getMessenger } from '../../messenger.js';
+import { getStorage } from '../../storage/index.js';
+import { getExtensionsPath } from './get-extensions-path.js';
+import { SyncStatus, getSyncStatus, setSyncStatus } from './sync-status.js';
+
+export const syncExtensions = async (): Promise<void> => {
+	const extensionsPath = getExtensionsPath();
+
+	if (!env['EXTENSIONS_LOCATION']) {
+		// Safe to run with multiple instances since dirs are created with `recursive: true`
+		return ensureExtensionDirs(extensionsPath, NESTED_EXTENSION_TYPES);
+	}
+
+	const messenger = getMessenger();
+
+	const isPrimaryProcess =
+		String(process.env['NODE_APP_INSTANCE']) === '0' || process.env['NODE_APP_INSTANCE'] === undefined;
+
+	const id = await mid.machineId();
+
+	const message = `extensions-sync/${id}`;
+
+	if (isPrimaryProcess === false) {
+		const isDone = (await getSyncStatus()) === SyncStatus.DONE;
+
+		if (isDone) return;
+
+		logger.trace('Extensions already being synced to this machine from another process.');
+
+		/**
+		 * Wait until the process that called the lock publishes a message that the syncing is complete
+		 */
+		return new Promise((resolve) => {
+			messenger.subscribe(message, () => resolve());
+		});
+	}
+
+	// Ensure that the local extensions cache path exists
+	await mkdir(extensionsPath, { recursive: true });
+	await setSyncStatus(SyncStatus.SYNCING);
+
+	logger.trace('Syncing extensions from configured storage location...');
+
+	const storage = await getStorage();
+
+	const disk = storage.location(env['EXTENSIONS_LOCATION']);
+
+	// Make sure we don't overload the file handles
+	const queue = new Queue({ concurrency: 1000 });
+
+	for await (const filepath of disk.list(env['EXTENSIONS_PATH'])) {
+		const readStream = await disk.read(filepath);
+
+		// We want files to be stored in the root of `$TEMP_PATH/extensions`, so gotta remove the
+		// extensions path on disk from the start of the file path
+		const destPath = join(extensionsPath, relative(resolve(sep, env['EXTENSIONS_PATH']), resolve(sep, filepath)));
+
+		// Ensure that the directory path exists
+		await mkdir(dirname(destPath), { recursive: true });
+
+		const writeStream = createWriteStream(destPath);
+
+		queue.add(() => pipeline(readStream, writeStream));
+	}
+
+	await queue.onIdle();
+
+	await ensureExtensionDirs(extensionsPath, NESTED_EXTENSION_TYPES);
+
+	await setSyncStatus(SyncStatus.DONE);
+	messenger.publish(message, { ready: true });
+};

api/src/extensions/lib/sync-status.ts

@@ -0,0 +1,29 @@
+import { exists } from 'fs-extra';
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { getExtensionsPath } from './get-extensions-path.js';
+
+export enum SyncStatus {
+	UNKNOWN = 'UNKNOWN',
+	SYNCING = 'SYNCING',
+	DONE = 'DONE',
+}
+
+/**
+ * Retrieves the sync status from the `.status` file in the local extensions folder
+ */
+export const getSyncStatus = async () => {
+	const statusFilePath = join(getExtensionsPath(), '.status');
+
+	if (await exists(statusFilePath)) {
+		const status = await readFile(statusFilePath, 'utf8');
+		return status;
+	} else {
+		return SyncStatus.UNKNOWN;
+	}
+};
+
+export const setSyncStatus = async (status: SyncStatus) => {
+	const statusFilePath = join(getExtensionsPath(), '.status');
+	await writeFile(statusFilePath, status);
+};

api/src/extensions/manager.ts

@@ -10,7 +10,7 @@ import type {
 	OperationApiConfig,
 } from '@directus/extensions';
 import { APP_SHARED_DEPS, HYBRID_EXTENSION_TYPES, NESTED_EXTENSION_TYPES } from '@directus/extensions';
-import { ensureExtensionDirs, generateExtensionsEntrypoint } from '@directus/extensions/node';
+import { generateExtensionsEntrypoint } from '@directus/extensions/node';
 import type {
 	ActionHandler,
 	EmbedHandler,
@@ -45,11 +45,13 @@ import { getSchema } from '../utils/get-schema.js';
 import { importFileUrl } from '../utils/import-file-url.js';
 import { JobQueue } from '../utils/job-queue.js';
 import { scheduleSynchronizedJob, validateCron } from '../utils/schedule.js';
+import { getExtensionsPath } from './lib/get-extensions-path.js';
 import { getExtensionsSettings } from './lib/get-extensions-settings.js';
 import { getExtensions } from './lib/get-extensions.js';
 import { getSharedDepsMapping } from './lib/get-shared-deps-mapping.js';
 import { generateApiExtensionsSandboxEntrypoint } from './lib/sandbox/generate-api-extensions-sandbox-entrypoint.js';
 import { instantiateSandboxSdk } from './lib/sandbox/sdk/instantiate.js';
+import { syncExtensions } from './lib/sync-extensions.js';
 import { wrapEmbeds } from './lib/wrap-embeds.js';
 import type { BundleConfig, ExtensionManagerOptions } from './types.js';
 
@@ -173,7 +175,7 @@ export class ExtensionManager {
 	 */
 	private async load(): Promise<void> {
 		try {
-			await ensureExtensionDirs(env['EXTENSIONS_PATH'], NESTED_EXTENSION_TYPES);
+			await syncExtensions();
 
 			this.extensions = await getExtensions();
 			this.extensionsSettings = await getExtensionsSettings(this.extensions);
@@ -290,7 +292,7 @@ export class ExtensionManager {
 	private initializeWatcher(): void {
 		logger.info('Watching extensions for changes...');
 
-		const extensionDirUrl = pathToRelativeUrl(env['EXTENSIONS_PATH']);
+		const extensionDirUrl = pathToRelativeUrl(getExtensionsPath());
 
 		const localExtensionUrls = NESTED_EXTENSION_TYPES.flatMap((type) => {
 			const typeDir = path.posix.join(extensionDirUrl, pluralize(type));

api/src/services/mail/index.ts

@@ -1,3 +1,4 @@
+import { InvalidPayloadError } from '@directus/errors';
 import type { Accountability, SchemaOverview } from '@directus/types';
 import fse from 'fs-extra';
 import type { Knex } from 'knex';
@@ -7,7 +8,7 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import getDatabase from '../../database/index.js';
 import env from '../../env.js';
-import { InvalidPayloadError } from '@directus/errors';
+import { getExtensionsPath } from '../../extensions/lib/get-extensions-path.js';
 import logger from '../../logger.js';
 import getMailer from '../../mailer.js';
 import type { AbstractServiceOptions } from '../../types/index.js';
@@ -16,7 +17,7 @@ import { Url } from '../../utils/url.js';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const liquidEngine = new Liquid({
-	root: [path.resolve(env['EXTENSIONS_PATH'], 'templates'), path.resolve(__dirname, 'templates')],
+	root: [path.resolve(getExtensionsPath(), 'templates'), path.resolve(__dirname, 'templates')],
 	extname: '.liquid',
 });
 
@@ -81,7 +82,7 @@ export class MailService {
 	}
 
 	private async renderTemplate(template: string, variables: Record<string, any>) {
-		const customTemplatePath = path.resolve(env['EXTENSIONS_PATH'], 'templates', template + '.liquid');
+		const customTemplatePath = path.resolve(getExtensionsPath(), 'templates', template + '.liquid');
 		const systemTemplatePath = path.join(__dirname, 'templates', template + '.liquid');
 
 		const templatePath = (await fse.pathExists(customTemplatePath)) ? customTemplatePath : systemTemplatePath;

api/src/utils/validate-storage.ts

@@ -1,9 +1,10 @@
-import env from '../env.js';
-import logger from '../logger.js';
-import { access } from 'node:fs/promises';
+import { toArray } from '@directus/utils';
 import { constants } from 'fs';
+import { access } from 'node:fs/promises';
 import path from 'path';
-import { toArray } from '@directus/utils';
+import env from '../env.js';
+import { getExtensionsPath } from '../extensions/lib/get-extensions-path.js';
+import logger from '../logger.js';
 
 export async function validateStorage(): Promise<void> {
 	if (env['DB_CLIENT'] === 'sqlite3') {
@@ -28,9 +29,11 @@ export async function validateStorage(): Promise<void> {
 		}
 	}
 
-	try {
-		await access(env['EXTENSIONS_PATH'], constants.R_OK);
-	} catch {
-		logger.warn(`Extensions directory (${path.resolve(env['EXTENSIONS_PATH'])}) is not readable!`);
+	if (!env['EXTENSIONS_LOCATION']) {
+		try {
+			await access(getExtensionsPath(), constants.R_OK);
+		} catch {
+			logger.warn(`Extensions directory (${path.resolve(getExtensionsPath())}) is not readable!`);
+		}
 	}
 }