Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Pthreads] Fix worker.js in ES6 module environments #21041

Merged
merged 7 commits into from
Jan 10, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ChangeLog.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works.

3.1.52 (in development)
-----------------------
- Building with `pthreads+EXPORT_ES6` will now emit the worker file as
`NAME.worker.mjs` rather than `.js`. This is a necessary breaking change to
resolve other `pthreads+EXPORT_ES6` issues in Node.js (because Node.js is
affected by the suffix in some cases). (#21041)
- Include paths added by ports (e.g. `-sUSE_SDL=2`) now use `-isystem` rather
then `-I`. This means that files in user-specified include directories will
now take precedence over port includes. (#21014)
Expand Down
18 changes: 17 additions & 1 deletion src/worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,16 @@ var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions
if (ENVIRONMENT_IS_NODE) {
// Create as web-worker-like an environment as we can.

// See the parallel code in shell.js, but here we don't need the condition on
// multi-environment builds, as we do not have the need to interact with the
// modularization logic as shell.js must (see link.py:node_es6_imports and
// how that is used in link.py).
#if EXPORT_ES6
const { createRequire } = await import('module');
/** @suppress{duplicate} */
var require = createRequire(import.meta.url);
#endif

var nodeWorkerThreads = require('worker_threads');

var parentPort = nodeWorkerThreads.parentPort;
Expand All @@ -32,7 +42,13 @@ if (ENVIRONMENT_IS_NODE) {
require,
Module,
location: {
href: __filename
// __filename is undefined in ES6 modules, and import.meta.url only in ES6
// modules.
#if EXPORT_ES6
href: typeof __filename !== 'undefined' ? __filename : import.meta.url
#else
href: typeof __filename
RReverser marked this conversation as resolved.
Show resolved Hide resolved
#endif
},
Worker: nodeWorkerThreads.Worker,
importScripts: (f) => vm.runInThisContext(fs.readFileSync(f, 'utf8'), {filename: f}),
Expand Down
21 changes: 16 additions & 5 deletions test/test_other.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,10 +343,10 @@ def test_emcc_output_worker_mjs(self, args):
test_file('hello_world.c')] + args)
src = read_file('subdir/hello_world.mjs')
self.assertContained("new URL('hello_world.wasm', import.meta.url)", src)
self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src)
self.assertContained('export default Module;', src)
src = read_file('subdir/hello_world.worker.js')
src = read_file('subdir/hello_world.worker.mjs')
self.assertContained("import('./hello_world.mjs')", src)
self.assertContained('hello, world!', self.run_js('subdir/hello_world.mjs'))

Expand All @@ -358,7 +358,7 @@ def test_emcc_output_worker_mjs_single_file(self):
test_file('hello_world.c'), '-sSINGLE_FILE'])
src = read_file('hello_world.mjs')
self.assertNotContained("new URL('data:", src)
self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src)
self.assertContained('hello, world!', self.run_js('hello_world.mjs'))

Expand Down Expand Up @@ -400,11 +400,16 @@ def test_export_es6_allows_export_in_post_js(self):
src = read_file('a.out.js')
self.assertContained('export{doNothing};', src)

@parameterized({
'': (False,),
'package_json': (True,),
})
@parameterized({
'': ([],),
'pthreads': (['-pthread'],),
# load a worker before startup to check ES6 modules there as well
'pthreads': (['-pthread', '-sPTHREAD_POOL_SIZE=1'],),
})
def test_export_es6(self, args):
def test_export_es6(self, args, package_json):
self.run_process([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6',
'-o', 'hello.mjs'] + args)
# In ES6 mode we use MODULARIZE, so we must instantiate an instance of the
Expand All @@ -413,6 +418,12 @@ def test_export_es6(self, args):
import Hello from "./hello.mjs";
Hello();
''')

if package_json:
# This makes node load all files in the directory as ES6 modules,
# including the worker.js file.
create_file('package.json', '{"type":"module"}')

self.assertContained('hello, world!', self.run_js('runner.mjs'))

def test_emcc_out_file(self):
Expand Down
36 changes: 23 additions & 13 deletions tools/link.py
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,10 @@ def do_split_module(wasm_file, options):
building.run_binaryen_command('wasm-split', wasm_file + '.orig', outfile=wasm_file, args=args)


def get_worker_js_suffix():
return '.worker.mjs' if settings.EXPORT_ES6 else '.worker.js'


def setup_pthreads(target):
if settings.RELOCATABLE:
# phtreads + dyanmic linking has certain limitations
Expand Down Expand Up @@ -569,7 +573,7 @@ def setup_pthreads(target):
building.user_requested_exports.update(worker_imports)

# set location of worker.js
settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js'
settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + get_worker_js_suffix()

if settings.MINIMAL_RUNTIME:
building.user_requested_exports.add('exit')
Expand Down Expand Up @@ -1999,12 +2003,27 @@ def phase_memory_initializer(memfile):
final_js += '.mem.js'


# Unmangle previously mangled `import.meta` and `await import` references in
# both main code and libraries.
# See also: `preprocess` in parseTools.js.
def fix_es6_import_statements(js_file):
if not settings.EXPORT_ES6 or not settings.USE_ES6_IMPORT_META:
return

src = read_file(js_file)
write_file(js_file, src
.replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
.replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))


def create_worker_file(input_file, target_dir, output_file):
output_file = os.path.join(target_dir, output_file)
input_file = utils.path_from_root(input_file)
contents = shared.read_and_preprocess(input_file, expand_macros=True)
write_file(output_file, contents)

fix_es6_import_statements(output_file)

# Minify the worker JS file, if JS minification is enabled.
if settings.MINIFY_WHITESPACE:
contents = building.acorn_optimizer(output_file, ['minifyWhitespace'], return_output=True)
Expand Down Expand Up @@ -2045,17 +2064,8 @@ def phase_final_emitting(options, state, target, wasm_target, memfile):
# mode)
final_js = building.closure_compiler(final_js, advanced=False, extra_closure_args=options.closure_args)

# Unmangle previously mangled `import.meta` and `await import` references in
# both main code and libraries.
# See also: `preprocess` in parseTools.js.
if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META:
src = read_file(final_js)
final_js += '.esmeta.js'
write_file(final_js, src
.replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
.replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))
shared.get_temp_files().note(final_js)
save_intermediate('es6-module')
fix_es6_import_statements(final_js)
save_intermediate('es6-module')

# Apply pre and postjs files
if options.extern_pre_js or options.extern_post_js:
Expand Down Expand Up @@ -2600,7 +2610,7 @@ def generate_worker_js(target, js_target, target_basename):
proxy_worker_filename = get_subresource_location(js_target)
else:
# compiler output goes in .worker.js file
move_file(js_target, shared.replace_suffix(js_target, '.worker.js'))
move_file(js_target, shared.replace_suffix(js_target, get_worker_js_suffix()))
worker_target_basename = target_basename + '.worker'
proxy_worker_filename = (settings.PROXY_TO_WORKER_FILENAME or worker_target_basename) + '.js'

Expand Down