From a73bb38a47b7bf59fae0c3b1ef1c1aa0a097d0aa Mon Sep 17 00:00:00 2001 From: Avi Vahl Date: Sun, 14 Jul 2024 18:54:18 +0300 Subject: [PATCH] feat: use node's native import() to load modules target is now node16, which means we still transpile our code to cjs (no "type": "module" yet). we now retain the dynamic import() calls as-is, rather than transpiling them to require() calls. this will make it easier for user projects to move to native esm. --- packages/engine-cli/src/cli.ts | 2 +- packages/engine-cli/src/load-config-file.ts | 4 ++-- packages/engine-cli/src/start-dev-server.ts | 3 ++- packages/engineer/src/cli-commands.ts | 3 ++- .../engineer/src/feature/gui.dev-server.env.ts | 9 +++++++-- packages/engineer/src/utils.ts | 3 ++- packages/runtime-node/src/dynamic-import.ts | 5 ----- packages/runtime-node/src/import-modules.ts | 3 +-- packages/runtime-node/src/index.ts | 8 ++++---- .../runtime-node/src/load-top-level-config.ts | 16 +++++++++++++--- packages/runtime-node/src/module-interop.ts | 18 ++++++++++++++++++ packages/runtime-node/src/node-environment.ts | 17 ++++++++++++----- .../load-features-from-paths.ts | 5 +++-- .../scripts/src/application/application.ts | 16 +++++++++++++--- packages/scripts/src/import-fresh.ts | 9 ++++++--- packages/scripts/src/resolve-exec-argv.ts | 10 ++++++++-- packages/scripts/src/run-environment.ts | 4 +++- tsconfig.base.json | 4 ++-- 18 files changed, 99 insertions(+), 40 deletions(-) delete mode 100644 packages/runtime-node/src/dynamic-import.ts create mode 100644 packages/runtime-node/src/module-interop.ts diff --git a/packages/engine-cli/src/cli.ts b/packages/engine-cli/src/cli.ts index e26324968..5eca9a3ec 100644 --- a/packages/engine-cli/src/cli.ts +++ b/packages/engine-cli/src/cli.ts @@ -1,5 +1,4 @@ import { cli, command } from 'cleye'; -import { analyzeCommand } from './analyze-command'; import type { EngineConfig } from '@wixc3/engine-scripts'; import { generateFeature } from './feature-generator'; import fs from '@file-services/node'; @@ -179,6 +178,7 @@ async function engine() { const rootDir = process.cwd(); if (argv.command === 'analyze') { + const { analyzeCommand } = await import('./analyze-command.js'); await analyzeCommand({ rootDir, feature: argv.flags.feature, engineConfig }); } else if (argv.command === 'generate') { const featureName = argv._.featureName; diff --git a/packages/engine-cli/src/load-config-file.ts b/packages/engine-cli/src/load-config-file.ts index 511331cbd..4b854bfb0 100644 --- a/packages/engine-cli/src/load-config-file.ts +++ b/packages/engine-cli/src/load-config-file.ts @@ -1,9 +1,9 @@ +import { getOriginalModule } from '@wixc3/engine-runtime-node'; import { pathToFileURL } from 'node:url'; -import { dynamicImport } from '@wixc3/engine-runtime-node'; export async function loadConfigFile(filePath: string): Promise { try { - const configModuleValue = (await dynamicImport(pathToFileURL(filePath))).default; + const configModuleValue = getOriginalModule(await import(pathToFileURL(filePath).href)); const config = (configModuleValue as { default: unknown }).default ?? configModuleValue; if (!config || typeof config !== 'object') { throw new Error(`config file: ${filePath} must export an object`); diff --git a/packages/engine-cli/src/start-dev-server.ts b/packages/engine-cli/src/start-dev-server.ts index 387cb8ac5..d813a15a0 100644 --- a/packages/engine-cli/src/start-dev-server.ts +++ b/packages/engine-cli/src/start-dev-server.ts @@ -1,6 +1,7 @@ import cors from 'cors'; import { safeListeningHttpServer } from 'create-listening-server'; import express from 'express'; +import type http from 'node:http'; import io from 'socket.io'; const noContentHandler: express.RequestHandler = (_req, res) => { @@ -27,7 +28,7 @@ export async function launchServer({ close: () => Promise; port: number; app: express.Express; - httpServer: import('http').Server; + httpServer: http.Server; socketServer: io.Server; }> { const app = express(); diff --git a/packages/engineer/src/cli-commands.ts b/packages/engineer/src/cli-commands.ts index c04924f31..87daf9512 100644 --- a/packages/engineer/src/cli-commands.ts +++ b/packages/engineer/src/cli-commands.ts @@ -9,6 +9,7 @@ import { parseCliArguments } from '@wixc3/engine-runtime-node'; import { Application } from '@wixc3/engine-scripts/dist/application/index.js'; import type { Command } from 'commander'; import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; import open from 'open'; import type { ServerListeningHandler } from './feature/dev-server.types.js'; import { startDevServer } from './utils.js'; @@ -265,7 +266,7 @@ async function preRequire(pathsToRequire: string[], basePath: string) { if (!resolvedFile) { throw new Error(`Cannot resolve "${request}"`); } - await import(resolvedFile); + await import(pathToFileURL(resolvedFile).href); } } diff --git a/packages/engineer/src/feature/gui.dev-server.env.ts b/packages/engineer/src/feature/gui.dev-server.env.ts index 8fed02a1a..5023e3982 100644 --- a/packages/engineer/src/feature/gui.dev-server.env.ts +++ b/packages/engineer/src/feature/gui.dev-server.env.ts @@ -1,8 +1,9 @@ import { nodeFs as fs } from '@file-services/node'; -import type { IConfigDefinition } from '@wixc3/engine-runtime-node'; +import { getOriginalModule, type IConfigDefinition } from '@wixc3/engine-runtime-node'; import { createMainEntrypoint, createVirtualEntries } from '@wixc3/engine-scripts'; import { SetMultiMap } from '@wixc3/patterns'; import HtmlWebpackPlugin from 'html-webpack-plugin'; +import { pathToFileURL } from 'node:url'; import type webpack from 'webpack'; import { devServerEnv } from './dev-server.feature.js'; import guiFeature, { mainDashboardEnv } from './gui.feature.js'; @@ -25,7 +26,11 @@ guiFeature.setup( const baseConfigPath = fs.findClosestFileSync(selfDirectoryPath, 'webpack.config.js'); const baseConfig = typeof baseConfigPath === 'string' - ? ((await import(baseConfigPath)) as { default: webpack.Configuration }).default + ? ( + getOriginalModule(await import(pathToFileURL(baseConfigPath).href)) as { + default: webpack.Configuration; + } + ).default : {}; const virtualModules: Record = {}; diff --git a/packages/engineer/src/utils.ts b/packages/engineer/src/utils.ts index f970481bd..21409e769 100644 --- a/packages/engineer/src/utils.ts +++ b/packages/engineer/src/utils.ts @@ -3,6 +3,7 @@ import { defaults } from '@wixc3/common'; import { BaseHost, RuntimeEngine, RuntimeFeature } from '@wixc3/engine-core'; import { runNodeEnvironment } from '@wixc3/engine-runtime-node'; import { isFeatureFile, loadFeaturesFromPaths, type EngineConfig } from '@wixc3/engine-scripts'; +import { pathToFileURL } from 'node:url'; import { TargetApplication } from './application-proxy-service.js'; import devServerFeature, { devServerEnv } from './feature/dev-server.feature.js'; import type { DevServerConfig } from './feature/dev-server.types.js'; @@ -91,6 +92,6 @@ function asDevConfig(options: DStartOptions, engineConfig: DEngineConfig): Parti async function preRequire(pathsToRequire: string[], basePath: string) { for (const request of pathsToRequire) { const resolvedRequest = require.resolve(request, { paths: [basePath] }); - await import(resolvedRequest); + await import(pathToFileURL(resolvedRequest).href); } } diff --git a/packages/runtime-node/src/dynamic-import.ts b/packages/runtime-node/src/dynamic-import.ts deleted file mode 100644 index 255c3231b..000000000 --- a/packages/runtime-node/src/dynamic-import.ts +++ /dev/null @@ -1,5 +0,0 @@ -// until we move to native esm, we need to use this hack to get dynamic imports to work -// eslint-disable-next-line @typescript-eslint/no-implied-eval -export const dynamicImport = new Function('modulePath', 'return import(modulePath);') as ( - modulePath: string | URL, -) => Promise<{ default: unknown }>; diff --git a/packages/runtime-node/src/import-modules.ts b/packages/runtime-node/src/import-modules.ts index b17d8c565..1d299e71d 100644 --- a/packages/runtime-node/src/import-modules.ts +++ b/packages/runtime-node/src/import-modules.ts @@ -1,5 +1,4 @@ import { pathToFileURL } from 'node:url'; -import { dynamicImport } from './dynamic-import'; /** * Dynamically imports required modules using the specified base path. @@ -10,7 +9,7 @@ import { dynamicImport } from './dynamic-import'; export async function importModules(basePath: string, requiredModules: string[]): Promise { for (const requiredModule of requiredModules) { try { - await dynamicImport(pathToFileURL(require.resolve(requiredModule, { paths: [basePath] }))); + await import(pathToFileURL(require.resolve(requiredModule, { paths: [basePath] })).href); } catch (ex) { throw new Error(`failed importing: ${requiredModule}`, { cause: ex }); } diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index 0e16794ca..d72318775 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -4,13 +4,16 @@ export * from './core-node/ipc-host.js'; export * from './core-node/local-env-inititializer.js'; export * from './core-node/parent-port-host.js'; export * from './core-node/ws-node-host.js'; -export * from './dynamic-import.js'; export * from './environments.js'; export * from './forked-process.js'; export * from './import-modules.js'; export * from './ipc-environment.js'; export * from './launch-http-server.js'; export * from './load-top-level-config.js'; +export * from './metadata-files.js'; +export * from './metrics-utils.js'; +export * from './micro-rpc.js'; +export * from './module-interop.js'; export * from './node-env-manager.js'; export * from './node-environment.js'; export * from './node-environments-manager.js'; @@ -20,6 +23,3 @@ export * from './types.js'; export * from './worker-thread-initializer.js'; export * from './worker-thread-initializer2.js'; export * from './ws-environment.js'; -export * from './metrics-utils.js'; -export * from './micro-rpc.js'; -export * from './metadata-files.js'; diff --git a/packages/runtime-node/src/load-top-level-config.ts b/packages/runtime-node/src/load-top-level-config.ts index a462dcfce..5ef7f0c9b 100644 --- a/packages/runtime-node/src/load-top-level-config.ts +++ b/packages/runtime-node/src/load-top-level-config.ts @@ -1,5 +1,7 @@ +import type { ConfigModule, TopLevelConfig } from '@wixc3/engine-core'; import type { SetMultiMap } from '@wixc3/patterns'; -import type { TopLevelConfig } from '@wixc3/engine-core'; +import { pathToFileURL } from 'node:url'; +import { getOriginalModule } from './module-interop'; import type { IConfigDefinition } from './types'; export async function loadTopLevelConfigs( @@ -24,11 +26,19 @@ export async function loadTopLevelConfigs( if (envName) { if (!definition.envName || definition.envName === envName) { config.push( - ...((await import(definition.filePath)) as { default: TopLevelConfig }).default, + ...( + getOriginalModule( + await import(pathToFileURL(definition.filePath).href), + ) as ConfigModule + ).default, ); } } else { - config.push(...((await import(definition.filePath)) as { default: TopLevelConfig }).default); + config.push( + ...( + getOriginalModule(await import(pathToFileURL(definition.filePath).href)) as ConfigModule + ).default, + ); } } catch (e) { throw new Error(`failed importing: ${definition.filePath}`, { cause: e }); diff --git a/packages/runtime-node/src/module-interop.ts b/packages/runtime-node/src/module-interop.ts new file mode 100644 index 000000000..6b42386d2 --- /dev/null +++ b/packages/runtime-node/src/module-interop.ts @@ -0,0 +1,18 @@ +/** + * When using native dynamic `import()` to evaluate a esm-as-cjs module, + * node messes up the module's shape, and the default export is nested under `default` property. + * This function is used to get the original module shape. + */ +export function getOriginalModule(moduleExports: unknown): unknown { + return isObjectLike(moduleExports) && + 'default' in moduleExports && + isObjectLike(moduleExports.default) && + '__esModule' in moduleExports.default && + !!moduleExports.default.__esModule + ? moduleExports.default + : moduleExports; +} + +function isObjectLike(value: unknown): value is object { + return typeof value === 'object' && value !== null; +} diff --git a/packages/runtime-node/src/node-environment.ts b/packages/runtime-node/src/node-environment.ts index 3cc4fa527..a3a7c2dca 100644 --- a/packages/runtime-node/src/node-environment.ts +++ b/packages/runtime-node/src/node-environment.ts @@ -8,6 +8,8 @@ import { type IFeatureLoader, type IPreloadModule, } from '@wixc3/engine-core'; +import { pathToFileURL } from 'node:url'; +import { getOriginalModule } from './module-interop.js'; import type { IEnvironmentDescriptor, IStaticFeatureDefinition, StartEnvironmentOptions } from './types.js'; export async function runNodeEnvironment({ @@ -92,7 +94,9 @@ export function createFeatureLoaders( const contextPreloadFilePath = preloadFilePaths[`${env.env}/${childEnvName}`]; if (contextPreloadFilePath) { - const preloadedContextModule = (await import(contextPreloadFilePath)) as IPreloadModule; + const preloadedContextModule = getOriginalModule( + await import(pathToFileURL(contextPreloadFilePath).href), + ) as IPreloadModule; if (preloadedContextModule.init) { initFunctions.push(preloadedContextModule.init); } @@ -100,7 +104,9 @@ export function createFeatureLoaders( } const preloadFilePath = preloadFilePaths[env.env]; if (preloadFilePath) { - const preloadedModule = (await import(preloadFilePath)) as IPreloadModule; + const preloadedModule = getOriginalModule( + await import(pathToFileURL(preloadFilePath).href), + ) as IPreloadModule; if (preloadedModule.init) { initFunctions.push(preloadedModule.init); } @@ -111,16 +117,17 @@ export function createFeatureLoaders( if (childEnvName && currentContext[env.env] === childEnvName) { const contextFilePath = contextFilePaths[`${env.env}/${childEnvName}`]; if (contextFilePath) { - await import(contextFilePath); + await import(pathToFileURL(contextFilePath).href); } } for (const { env: envName } of new Set([env, ...env.dependencies])) { const envFilePath = envFilePaths[envName]; if (envFilePath) { - await import(envFilePath); + await import(pathToFileURL(envFilePath).href); } } - return ((await import(filePath)) as { default: FeatureClass }).default; + return (getOriginalModule(await import(pathToFileURL(filePath).href)) as { default: FeatureClass }) + .default; }, depFeatures: dependencies, resolvedContexts, diff --git a/packages/scripts/src/analyze-feature/load-features-from-paths.ts b/packages/scripts/src/analyze-feature/load-features-from-paths.ts index d2d5940ca..bbe4e0127 100644 --- a/packages/scripts/src/analyze-feature/load-features-from-paths.ts +++ b/packages/scripts/src/analyze-feature/load-features-from-paths.ts @@ -1,9 +1,10 @@ import type { IFileSystemSync } from '@file-services/types'; import { concat, getValue, isPlainObject, map } from '@wixc3/common'; import type { FeatureClass } from '@wixc3/engine-core'; -import type { IConfigDefinition } from '@wixc3/engine-runtime-node'; +import { getOriginalModule, type IConfigDefinition } from '@wixc3/engine-runtime-node'; import { SetMultiMap } from '@wixc3/patterns'; import type { INpmPackage } from '@wixc3/resolve-directory-context'; +import { pathToFileURL } from 'node:url'; import { isFeatureFile, parseConfigFileName, @@ -119,7 +120,7 @@ function setEnvPath( } async function analyzeFeature(filePath: string, featurePackage: IPackageDescriptor): Promise { - const moduleExports = await import(filePath); + const moduleExports = getOriginalModule(await import(pathToFileURL(filePath).href)); const module = analyzeFeatureModule(filePath, moduleExports); const scopedName = scopeToPackage(featurePackage.simplifiedName, module.name); return { diff --git a/packages/scripts/src/application/application.ts b/packages/scripts/src/application/application.ts index f425a65fd..682facd93 100644 --- a/packages/scripts/src/application/application.ts +++ b/packages/scripts/src/application/application.ts @@ -2,6 +2,7 @@ import { nodeFs as fs } from '@file-services/node'; import { defaults } from '@wixc3/common'; import { type TopLevelConfig } from '@wixc3/engine-core'; import { + getOriginalModule, launchEngineHttpServer, NodeEnvironmentsManager, resolveEnvironments, @@ -10,6 +11,7 @@ import { } from '@wixc3/engine-runtime-node'; import { createDisposables, SetMultiMap } from '@wixc3/patterns'; import express from 'express'; +import { pathToFileURL } from 'node:url'; import webpack from 'webpack'; import { analyzeFeatures } from '../analyze-feature'; import { ENGINE_CONFIG_FILE_NAME } from '../build-constants'; @@ -222,7 +224,11 @@ export class Application { if (engineConfigFilePath) { try { return { - config: ((await import(engineConfigFilePath)) as { default: EngineConfig }).default, + config: ( + getOriginalModule(await import(pathToFileURL(engineConfigFilePath).href)) as { + default: EngineConfig; + } + ).default, path: engineConfigFilePath, }; } catch (ex) { @@ -239,7 +245,7 @@ export class Application { protected async importModules(requiredModules: string[]) { for (const requiredModule of requiredModules) { try { - await import(require.resolve(requiredModule, { paths: [this.basePath] })); + await import(pathToFileURL(require.resolve(requiredModule, { paths: [this.basePath] })).href); } catch (ex) { throw new Error(`failed importing: ${requiredModule}`, { cause: ex }); } @@ -409,7 +415,11 @@ export class Application { : fs.findClosestFileSync(basePath, 'webpack.config.js'); const baseConfig = typeof baseConfigPath === 'string' - ? ((await import(baseConfigPath)) as { default: webpack.Configuration }).default + ? ( + getOriginalModule(await import(pathToFileURL(baseConfigPath).href)) as { + default: webpack.Configuration; + } + ).default : {}; const webpackConfigs = createWebpackConfigs({ baseConfig, diff --git a/packages/scripts/src/import-fresh.ts b/packages/scripts/src/import-fresh.ts index 9c0c1a033..34b7f3840 100644 --- a/packages/scripts/src/import-fresh.ts +++ b/packages/scripts/src/import-fresh.ts @@ -1,4 +1,6 @@ +import { getOriginalModule } from '@wixc3/engine-runtime-node'; import { once } from 'node:events'; +import { pathToFileURL } from 'node:url'; import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'; /** @@ -28,7 +30,7 @@ if (!isMainThread && isImportWorkerData(workerData)) { if (Array.isArray(filePath)) { const imported: Promise[] = []; for (const path of filePath) { - imported.push(import(path)); + imported.push(import(pathToFileURL(path).href).then(getOriginalModule)); } Promise.all(imported) .then((moduleExports) => { @@ -42,8 +44,9 @@ if (!isMainThread && isImportWorkerData(workerData)) { throw e; }); } else { - import(filePath) - .then((moduleExports) => parentPort?.postMessage(moduleExports[exportSymbolName])) + import(pathToFileURL(filePath).href) + .then(getOriginalModule) + .then((moduleExports: any) => parentPort?.postMessage(moduleExports[exportSymbolName])) .catch((e) => { throw e; }); diff --git a/packages/scripts/src/resolve-exec-argv.ts b/packages/scripts/src/resolve-exec-argv.ts index 9fdebf26d..da92fb76b 100644 --- a/packages/scripts/src/resolve-exec-argv.ts +++ b/packages/scripts/src/resolve-exec-argv.ts @@ -1,10 +1,16 @@ import { nodeFs as fs } from '@file-services/node'; -import type { EngineConfig } from './types'; +import { getOriginalModule } from '@wixc3/engine-runtime-node'; +import { pathToFileURL } from 'node:url'; import { ENGINE_CONFIG_FILE_NAME } from './build-constants'; +import type { EngineConfig } from './types'; export async function resolveExecArgv(basePath: string) { const engineConfig = await fs.promises.findClosestFile(basePath, ENGINE_CONFIG_FILE_NAME); - const { default: config } = (engineConfig ? await import(engineConfig) : {}) as { default?: EngineConfig }; + const { default: config } = ( + engineConfig ? getOriginalModule(await import(pathToFileURL(engineConfig).href)) : {} + ) as { + default?: EngineConfig; + }; const execArgv = [...process.execArgv]; if (config?.require) { diff --git a/packages/scripts/src/run-environment.ts b/packages/scripts/src/run-environment.ts index a5d37d06c..310e1406c 100644 --- a/packages/scripts/src/run-environment.ts +++ b/packages/scripts/src/run-environment.ts @@ -15,10 +15,12 @@ import { IStaticFeatureDefinition, METADATA_PROVIDER_ENV_ID, MetadataCollectionAPI, + getOriginalModule, loadTopLevelConfigs, metadataApiToken, runNodeEnvironment, } from '@wixc3/engine-runtime-node'; +import { pathToFileURL } from 'node:url'; import { findFeatures } from './analyze-feature/index.js'; import { ENGINE_CONFIG_FILE_NAME } from './build-constants.js'; import { EngineConfig, IFeatureDefinition } from './types.js'; @@ -218,7 +220,7 @@ export async function getRunningFeature { try { - return ((await import(filePath)) as { default: unknown }).default; + return (getOriginalModule(await import(pathToFileURL(filePath).href)) as { default: unknown }).default; } catch (ex) { throw new Error(`failed importing file: ${filePath}`, { cause: ex }); } diff --git a/tsconfig.base.json b/tsconfig.base.json index d58dbc8e2..b8a18145e 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -25,9 +25,9 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "commonjs", /* Specify what module code is generated. */ + "module": "node16", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ - "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "node16", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */