Skip to content

esm: populate separate cache for require(esm) in imported CJS #59679

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

esm: populate separate cache for require(esm) in imported CJS #59679

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 80 additions & 58 deletions lib/internal/modules/cjs/loader.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ const {
module_export_names_private_symbol,
module_circular_visited_private_symbol,
module_export_private_symbol,
module_parent_private_symbol,
module_first_parent_private_symbol,
module_last_parent_private_symbol,
},
isInsideNodeModules,
} = internalBinding('util');
Expand All @@ -94,9 +95,13 @@ const kModuleCircularVisited = module_circular_visited_private_symbol;
*/
const kModuleExport = module_export_private_symbol;
/**
* {@link Module} parent module.
* {@link Module} The first parent module that loads a module with require().
*/
const kModuleParent = module_parent_private_symbol;
const kFirstModuleParent = module_first_parent_private_symbol;
/**
* {@link Module} The last parent module that loads a module with require().
*/
const kLastModuleParent = module_last_parent_private_symbol;

const kIsMainSymbol = Symbol('kIsMainSymbol');
const kIsCachedByESMLoader = Symbol('kIsCachedByESMLoader');
Expand All @@ -117,6 +122,7 @@ module.exports = {
findLongestRegisteredExtension,
resolveForCJSWithHooks,
loadSourceForCJSWithHooks: loadSource,
populateCJSExportsFromESM,
wrapSafe,
wrapModuleLoad,
kIsMainSymbol,
Expand Down Expand Up @@ -326,7 +332,8 @@ function Module(id = '', parent) {
this.id = id;
this.path = path.dirname(id);
setOwnProperty(this, 'exports', {});
this[kModuleParent] = parent;
this[kFirstModuleParent] = parent;
this[kLastModuleParent] = parent;
updateChildren(parent, this, false);
this.filename = null;
this.loaded = false;
Expand Down Expand Up @@ -408,7 +415,7 @@ ObjectDefineProperty(BuiltinModule.prototype, 'isPreloading', isPreloadingDesc);
* @returns {object}
*/
function getModuleParent() {
return this[kModuleParent];
return this[kFirstModuleParent];
}

/**
Expand All @@ -418,7 +425,7 @@ function getModuleParent() {
* @returns {void}
*/
function setModuleParent(value) {
this[kModuleParent] = value;
this[kFirstModuleParent] = value;
}

let debug = debuglog('module', (fn) => {
Expand Down Expand Up @@ -997,7 +1004,7 @@ function getExportsForCircularRequire(module) {
const requiredESM = module[kRequiredModuleSymbol];
if (requiredESM && requiredESM.getStatus() !== kEvaluated) {
let message = `Cannot require() ES Module ${module.id} in a cycle.`;
const parent = module[kModuleParent];
const parent = module[kLastModuleParent];
if (parent) {
message += ` (from ${parent.filename})`;
}
Expand Down Expand Up @@ -1278,6 +1285,8 @@ Module._load = function(request, parent, isMain) {
// load hooks for the module keyed by the (potentially customized) filename.
module[kURL] = url;
module[kFormat] = format;
} else {
module[kLastModuleParent] = parent;
}

if (parent !== undefined) {
Expand Down Expand Up @@ -1397,7 +1406,8 @@ Module._resolveFilename = function(request, parent, isMain, options) {
const requireStack = [];
for (let cursor = parent;
cursor;
cursor = cursor[kModuleParent]) {
// TODO(joyeecheung): it makes more sense to use kLastModuleParent here.
cursor = cursor[kFirstModuleParent]) {
ArrayPrototypePush(requireStack, cursor.filename || cursor.id);
}
let message = `Cannot find module '${request}'`;
Expand Down Expand Up @@ -1514,7 +1524,7 @@ function loadESMFromCJS(mod, filename, format, source) {
// ESM won't be accessible via process.mainModule.
setOwnProperty(process, 'mainModule', undefined);
} else {
const parent = mod[kModuleParent];
const parent = mod[kLastModuleParent];

requireModuleWarningMode ??= getOptionValue('--trace-require-module');
if (requireModuleWarningMode) {
Expand Down Expand Up @@ -1564,54 +1574,66 @@ function loadESMFromCJS(mod, filename, format, source) {
wrap,
namespace,
} = cascadedLoader.importSyncForRequire(mod, filename, source, isMain, parent);
// Tooling in the ecosystem have been using the __esModule property to recognize
// transpiled ESM in consuming code. For example, a 'log' package written in ESM:
//
// export default function log(val) { console.log(val); }
//
// Can be transpiled as:
//
// exports.__esModule = true;
// exports.default = function log(val) { console.log(val); }
//
// The consuming code may be written like this in ESM:
//
// import log from 'log'
//
// Which gets transpiled to:
//
// const _mod = require('log');
// const log = _mod.__esModule ? _mod.default : _mod;
//
// So to allow transpiled consuming code to recognize require()'d real ESM
// as ESM and pick up the default exports, we add a __esModule property by
// building a source text module facade for any module that has a default
// export and add .__esModule = true to the exports. This maintains the
// enumerability of the re-exported names and the live binding of the exports,
// without incurring a non-trivial per-access overhead on the exports.
//
// The source of the facade is defined as a constant per-isolate property
// required_module_default_facade_source_string, which looks like this
//
// export * from 'original';
// export { default } from 'original';
// export const __esModule = true;
//
// And the 'original' module request is always resolved by
// createRequiredModuleFacade() to `wrap` which is a ModuleWrap wrapping
// over the original module.

// We don't do this to modules that are marked as CJS ESM or that
// don't have default exports to avoid the unnecessary overhead.
// If __esModule is already defined, we will also skip the extension
// to allow users to override it.
if (ObjectHasOwn(namespace, 'module.exports')) {
mod.exports = namespace['module.exports'];
} else if (!ObjectHasOwn(namespace, 'default') || ObjectHasOwn(namespace, '__esModule')) {
mod.exports = namespace;
} else {
mod.exports = createRequiredModuleFacade(wrap);
}

populateCJSExportsFromESM(mod, wrap, namespace);
}
}

/**
* Populate the exports of a CJS module entry from an ESM module's namespace object for
* require(esm).
* @param {Module} mod CJS module instance
* @param {ModuleWrap} wrap ESM ModuleWrap instance.
* @param {object} namespace The ESM namespace object.
*/
function populateCJSExportsFromESM(mod, wrap, namespace) {
// Tooling in the ecosystem have been using the __esModule property to recognize
// transpiled ESM in consuming code. For example, a 'log' package written in ESM:
//
// export default function log(val) { console.log(val); }
//
// Can be transpiled as:
//
// exports.__esModule = true;
// exports.default = function log(val) { console.log(val); }
//
// The consuming code may be written like this in ESM:
//
// import log from 'log'
//
// Which gets transpiled to:
//
// const _mod = require('log');
// const log = _mod.__esModule ? _mod.default : _mod;
//
// So to allow transpiled consuming code to recognize require()'d real ESM
// as ESM and pick up the default exports, we add a __esModule property by
// building a source text module facade for any module that has a default
// export and add .__esModule = true to the exports. This maintains the
// enumerability of the re-exported names and the live binding of the exports,
// without incurring a non-trivial per-access overhead on the exports.
//
// The source of the facade is defined as a constant per-isolate property
// required_module_default_facade_source_string, which looks like this
//
// export * from 'original';
// export { default } from 'original';
// export const __esModule = true;
//
// And the 'original' module request is always resolved by
// createRequiredModuleFacade() to `wrap` which is a ModuleWrap wrapping
// over the original module.

// We don't do this to modules that are marked as CJS ESM or that
// don't have default exports to avoid the unnecessary overhead.
// If __esModule is already defined, we will also skip the extension
// to allow users to override it.
if (ObjectHasOwn(namespace, 'module.exports')) {
mod.exports = namespace['module.exports'];
} else if (!ObjectHasOwn(namespace, 'default') || ObjectHasOwn(namespace, '__esModule')) {
mod.exports = namespace;
} else {
mod.exports = createRequiredModuleFacade(wrap);
}
}

Expand Down Expand Up @@ -1804,7 +1826,7 @@ function reconstructErrorStack(err, parentPath, parentSource) {
*/
function getRequireESMError(mod, pkg, content, filename) {
// This is an error path because `require` of a `.js` file in a `"type": "module"` scope is not allowed.
const parent = mod[kModuleParent];
const parent = mod[kFirstModuleParent];
const parentPath = parent?.filename;
const packageJsonPath = pkg?.path;
const usesEsm = containsModuleSyntax(content, filename);
Expand Down
47 changes: 33 additions & 14 deletions lib/internal/modules/esm/loader.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const {
const {
kIsExecuting,
kRequiredModuleSymbol,
Module: CJSModule,
} = require('internal/modules/cjs/loader');
const { imported_cjs_symbol } = internalBinding('symbols');

Expand Down Expand Up @@ -91,13 +92,18 @@ function newLoadCache() {
return new LoadCache();
}

let _translators;
function lazyLoadTranslators() {
_translators ??= require('internal/modules/esm/translators');
return _translators;
}

/**
* Lazy-load translators to avoid potentially unnecessary work at startup (ex if ESM is not used).
* @returns {import('./translators.js').Translators}
*/
function getTranslators() {
const { translators } = require('internal/modules/esm/translators');
return translators;
return lazyLoadTranslators().translators;
}

/**
Expand Down Expand Up @@ -506,7 +512,7 @@ class ModuleLoader {

const { source } = loadResult;
const isMain = (parentURL === undefined);
const wrap = this.#translate(url, finalFormat, source, isMain);
const wrap = this.#translate(url, finalFormat, source, parentURL);
assert(wrap instanceof ModuleWrap, `Translator used for require(${url}) should not be async`);

if (process.env.WATCH_REPORT_DEPENDENCIES && process.send) {
Expand Down Expand Up @@ -542,18 +548,31 @@ class ModuleLoader {
* @param {string} format Format of the module to be translated. This is used to find
* matching translators.
* @param {ModuleSource} source Source of the module to be translated.
* @param {boolean} isMain Whether the module to be translated is the entry point.
* @param {string|undefined} parentURL URL of the parent module. Undefined if it's the entry point.
* @returns {ModuleWrap}
*/
#translate(url, format, source, isMain) {
#translate(url, format, source, parentURL) {
this.validateLoadResult(url, format);
const translator = getTranslators().get(format);

if (!translator) {
throw new ERR_UNKNOWN_MODULE_FORMAT(format, url);
}

const result = FunctionPrototypeCall(translator, this, url, source, isMain);
// Populate the CJS cache with a facade for ESM in case subsequent require(esm) is
// looking it up from the cache. The parent module of the CJS cache entry would be the
// first CJS module that loads it with require(). This is an approximation, because
// ESM caches more and it does not get re-loaded and updated every time an `import` is
// encountered, unlike CJS require(), and we only use the parent entry to provide
// more information in error messages.
if (format === 'module') {
const parentFilename = urlToFilename(parentURL);
const parent = parentFilename ? CJSModule._cache[parentFilename] : undefined;
const cjsModule = lazyLoadTranslators().cjsEmplaceModuleCacheEntryForURL(url, parent);
debug('cjsEmplaceModuleCacheEntryForURL', url, parent, cjsModule);
}

const result = FunctionPrototypeCall(translator, this, url, source, parentURL === undefined);
assert(result instanceof ModuleWrap);
return result;
}
Expand All @@ -563,10 +582,10 @@ class ModuleLoader {
* This is run synchronously, and the translator always return a ModuleWrap synchronously.
* @param {string} url URL of the module to be translated.
* @param {object} loadContext See {@link load}
* @param {boolean} isMain Whether the module to be translated is the entry point.
* @param {string|undefined} parentURL URL of the parent module. Undefined if it's the entry point.
* @returns {ModuleWrap}
*/
loadAndTranslateForRequireInImportedCJS(url, loadContext, isMain) {
loadAndTranslateForRequireInImportedCJS(url, loadContext, parentURL) {
const { format: formatFromLoad, source } = this.#loadSync(url, loadContext);

if (formatFromLoad === 'wasm') { // require(wasm) is not supported.
Expand All @@ -587,7 +606,7 @@ class ModuleLoader {
finalFormat = 'require-commonjs-typescript';
}

const wrap = this.#translate(url, finalFormat, source, isMain);
const wrap = this.#translate(url, finalFormat, source, parentURL);
assert(wrap instanceof ModuleWrap, `Translator used for require(${url}) should not be async`);
return wrap;
}
Expand All @@ -597,13 +616,13 @@ class ModuleLoader {
* This may be run asynchronously if there are asynchronous module loader hooks registered.
* @param {string} url URL of the module to be translated.
* @param {object} loadContext See {@link load}
* @param {boolean} isMain Whether the module to be translated is the entry point.
* @param {string|undefined} parentURL URL of the parent module. Undefined if it's the entry point.
* @returns {Promise<ModuleWrap>|ModuleWrap}
*/
loadAndTranslate(url, loadContext, isMain) {
loadAndTranslate(url, loadContext, parentURL) {
const maybePromise = this.load(url, loadContext);
const afterLoad = ({ format, source }) => {
return this.#translate(url, format, source, isMain);
return this.#translate(url, format, source, parentURL);
};
if (isPromise(maybePromise)) {
return maybePromise.then(afterLoad);
Expand All @@ -630,9 +649,9 @@ class ModuleLoader {
const isMain = parentURL === undefined;
let moduleOrModulePromise;
if (isForRequireInImportedCJS) {
moduleOrModulePromise = this.loadAndTranslateForRequireInImportedCJS(url, context, isMain);
moduleOrModulePromise = this.loadAndTranslateForRequireInImportedCJS(url, context, parentURL);
} else {
moduleOrModulePromise = this.loadAndTranslate(url, context, isMain);
moduleOrModulePromise = this.loadAndTranslate(url, context, parentURL);
}

const inspectBrk = (
Expand Down
Loading
Loading