From 49ab23ad9f636377eb1ee227445b95040da4afdb Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 10 Jan 2024 12:15:52 -0800 Subject: [PATCH] [Pthreads] Fix worker.js in ES6 module environments (#21041) This file can be an ES6 module if a package.json file indicates that all files in the directory are. We need to apply the same tricks as we apply to the main JS file in that case, with createRequire etc. We also need to emit the suffix .worker.mjs and not .js, or else node will error. Followup to #20939 and fixes the package.json discussion after that PR landed. --- ChangeLog.md | 4 ++++ src/worker.js | 16 ++++++++++++++++ test/test_other.py | 21 ++++++++++++++++----- tools/link.py | 36 +++++++++++++++++++++++------------- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 757355b940f2..7bc619cc29ba 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works. 3.1.52 (in development) ----------------------- +- Building with `pthreads+EXPORT_ES6` will now emit the worker file as + `NAME.worker.mjs` rather than `.js`. This is a necessary breaking change to + resolve other `pthreads+EXPORT_ES6` issues in Node.js (because Node.js is + affected by the suffix in some cases). (#21041) - Include paths added by ports (e.g. `-sUSE_SDL=2`) now use `-isystem` rather then `-I`. This means that files in user-specified include directories will now take precedence over port includes. (#21014) diff --git a/src/worker.js b/src/worker.js index 425d084e9508..0b04ceedc227 100644 --- a/src/worker.js +++ b/src/worker.js @@ -18,6 +18,16 @@ var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions if (ENVIRONMENT_IS_NODE) { // Create as web-worker-like an environment as we can. + // See the parallel code in shell.js, but here we don't need the condition on + // multi-environment builds, as we do not have the need to interact with the + // modularization logic as shell.js must (see link.py:node_es6_imports and + // how that is used in link.py). +#if EXPORT_ES6 + const { createRequire } = await import('module'); + /** @suppress{duplicate} */ + var require = createRequire(import.meta.url); +#endif + var nodeWorkerThreads = require('worker_threads'); var parentPort = nodeWorkerThreads.parentPort; @@ -32,7 +42,13 @@ if (ENVIRONMENT_IS_NODE) { require, Module, location: { + // __filename is undefined in ES6 modules, and import.meta.url only in ES6 + // modules. +#if EXPORT_ES6 + href: typeof __filename !== 'undefined' ? __filename : import.meta.url +#else href: __filename +#endif }, Worker: nodeWorkerThreads.Worker, importScripts: (f) => vm.runInThisContext(fs.readFileSync(f, 'utf8'), {filename: f}), diff --git a/test/test_other.py b/test/test_other.py index 01660c2a702b..45daf7d9022f 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -343,10 +343,10 @@ def test_emcc_output_worker_mjs(self, args): test_file('hello_world.c')] + args) src = read_file('subdir/hello_world.mjs') self.assertContained("new URL('hello_world.wasm', import.meta.url)", src) - self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src) + self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src) self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src) self.assertContained('export default Module;', src) - src = read_file('subdir/hello_world.worker.js') + src = read_file('subdir/hello_world.worker.mjs') self.assertContained("import('./hello_world.mjs')", src) self.assertContained('hello, world!', self.run_js('subdir/hello_world.mjs')) @@ -358,7 +358,7 @@ def test_emcc_output_worker_mjs_single_file(self): test_file('hello_world.c'), '-sSINGLE_FILE']) src = read_file('hello_world.mjs') self.assertNotContained("new URL('data:", src) - self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src) + self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src) self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src) self.assertContained('hello, world!', self.run_js('hello_world.mjs')) @@ -400,11 +400,16 @@ def test_export_es6_allows_export_in_post_js(self): src = read_file('a.out.js') self.assertContained('export{doNothing};', src) + @parameterized({ + '': (False,), + 'package_json': (True,), + }) @parameterized({ '': ([],), - 'pthreads': (['-pthread'],), + # load a worker before startup to check ES6 modules there as well + 'pthreads': (['-pthread', '-sPTHREAD_POOL_SIZE=1'],), }) - def test_export_es6(self, args): + def test_export_es6(self, args, package_json): self.run_process([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6', '-o', 'hello.mjs'] + args) # In ES6 mode we use MODULARIZE, so we must instantiate an instance of the @@ -413,6 +418,12 @@ def test_export_es6(self, args): import Hello from "./hello.mjs"; Hello(); ''') + + if package_json: + # This makes node load all files in the directory as ES6 modules, + # including the worker.js file. + create_file('package.json', '{"type":"module"}') + self.assertContained('hello, world!', self.run_js('runner.mjs')) def test_emcc_out_file(self): diff --git a/tools/link.py b/tools/link.py index 8cd3c02c810c..e3f206ff0816 100644 --- a/tools/link.py +++ b/tools/link.py @@ -513,6 +513,10 @@ def do_split_module(wasm_file, options): building.run_binaryen_command('wasm-split', wasm_file + '.orig', outfile=wasm_file, args=args) +def get_worker_js_suffix(): + return '.worker.mjs' if settings.EXPORT_ES6 else '.worker.js' + + def setup_pthreads(target): if settings.RELOCATABLE: # phtreads + dyanmic linking has certain limitations @@ -569,7 +573,7 @@ def setup_pthreads(target): building.user_requested_exports.update(worker_imports) # set location of worker.js - settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js' + settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + get_worker_js_suffix() if settings.MINIMAL_RUNTIME: building.user_requested_exports.add('exit') @@ -1999,12 +2003,27 @@ def phase_memory_initializer(memfile): final_js += '.mem.js' +# Unmangle previously mangled `import.meta` and `await import` references in +# both main code and libraries. +# See also: `preprocess` in parseTools.js. +def fix_es6_import_statements(js_file): + if not settings.EXPORT_ES6 or not settings.USE_ES6_IMPORT_META: + return + + src = read_file(js_file) + write_file(js_file, src + .replace('EMSCRIPTEN$IMPORT$META', 'import.meta') + .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import')) + + def create_worker_file(input_file, target_dir, output_file): output_file = os.path.join(target_dir, output_file) input_file = utils.path_from_root(input_file) contents = shared.read_and_preprocess(input_file, expand_macros=True) write_file(output_file, contents) + fix_es6_import_statements(output_file) + # Minify the worker JS file, if JS minification is enabled. if settings.MINIFY_WHITESPACE: contents = building.acorn_optimizer(output_file, ['minifyWhitespace'], return_output=True) @@ -2045,17 +2064,8 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): # mode) final_js = building.closure_compiler(final_js, advanced=False, extra_closure_args=options.closure_args) - # Unmangle previously mangled `import.meta` and `await import` references in - # both main code and libraries. - # See also: `preprocess` in parseTools.js. - if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META: - src = read_file(final_js) - final_js += '.esmeta.js' - write_file(final_js, src - .replace('EMSCRIPTEN$IMPORT$META', 'import.meta') - .replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import')) - shared.get_temp_files().note(final_js) - save_intermediate('es6-module') + fix_es6_import_statements(final_js) + save_intermediate('es6-module') # Apply pre and postjs files if options.extern_pre_js or options.extern_post_js: @@ -2600,7 +2610,7 @@ def generate_worker_js(target, js_target, target_basename): proxy_worker_filename = get_subresource_location(js_target) else: # compiler output goes in .worker.js file - move_file(js_target, shared.replace_suffix(js_target, '.worker.js')) + move_file(js_target, shared.replace_suffix(js_target, get_worker_js_suffix())) worker_target_basename = target_basename + '.worker' proxy_worker_filename = (settings.PROXY_TO_WORKER_FILENAME or worker_target_basename) + '.js'