Merge pull request #974 from messense/emscripten

Add support for `wasm32-unknown-emscripten` target

.github/workflows/test.yml

@@ -148,6 +148,39 @@ jobs:
           export PYO3_CONFIG_FILE=$(pwd)/test-crates/pyo3-mixed/pyo3-config.txt
           cargo run -- build -m test-crates/pyo3-mixed/Cargo.toml --target x86_64-unknown-linux-gnu --zig
 
+  test-emscripten:
+    name: Test Emscripten
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          components: rust-src
+          target: wasm32-unknown-emscripten
+          override: true
+      - uses: mymindstorm/setup-emsdk@v11
+        with:
+          version: 3.1.13
+          actions-cache-folder: emsdk-cache
+      - uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: "3.10"
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - run: pip install nox
+      - uses: actions/cache@v3
+        with:
+          path: |
+            tests/pyodide
+          key: ${{ hashFiles('tests/*.js') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }}
+      - uses: Swatinem/rust-cache@v1
+      - name: Run tests
+        run: nox -s test-emscripten
+
   test-alpine:
     name: Test Alpine Linux
     runs-on: ubuntu-latest

.gitignore

@@ -14,3 +14,4 @@ tags
 test-crates/wheels/
 test-crates/targets/
 test-crates/venvs/
+tests/pyodide/

Changelog.md

@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add `--find-interpreter` option to `build` and `publish` commands to search for python interpreters in [#964](https://github.com/PyO3/maturin/pull/964)
 * Infer target triple from `ARCHFLAGS` for macOS to be compatible with `cibuildwheel` in [#967](https://github.com/PyO3/maturin/pull/967)
 * Expose commonly used Cargo CLI options in `maturin build` command in [#972](https://github.com/PyO3/maturin/pull/972)
+* Add support for `wasm32-unknown-emscripten` target in [#974](https://github.com/PyO3/maturin/pull/974)
 
 ## [0.12.20] - 2022-06-15
 

MANIFEST.in

@@ -1,7 +1,7 @@
 include Cargo.toml Cargo.lock
 include Readme.md
 include license-apache license-mit
-recursive-include src *.rs
+recursive-include src *.rs *.py
 recursive-include src/auditwheel *.json
 recursive-include src/python_interpreter *.py *.json
 recursive-include src/templates *.j2

noxfile.py

@@ -0,0 +1,54 @@
+import sys
+from pathlib import Path
+
+import nox
+
+
+def download_pyodide(session: nox.Session, pyodide_dir: Path) -> None:
+    pyodide_dir.mkdir()
+
+    PYODIDE_DEV = "https://pyodide-cdn2.iodide.io/dev/full/"
+    pyodide_files = [
+        "pyodide.js",
+        "packages.json",
+        "pyodide.asm.js",
+        "pyodide.asm.data",
+        "pyodide.asm.wasm",
+        "pyodide_py.tar",
+    ]
+    with session.chdir(pyodide_dir):
+        for file in pyodide_files:
+            session.run("wget", "-q", PYODIDE_DEV + file, external=True)
+        session.run("npm", "i", "node-fetch", external=True)
+
+
+@nox.session(name="test-emscripten")
+def test_emscripten(session: nox.Session):
+    emscripten_dir = Path("./tests").resolve()
+    pyodide_dir = emscripten_dir / "pyodide"
+    if not pyodide_dir.exists():
+        download_pyodide(session, pyodide_dir)
+
+    test_crates = [
+        "test-crates/pyo3-mixed",
+    ]
+    for crate in test_crates:
+        crate = Path(crate).resolve()
+
+        ver = sys.version_info
+        session.run(
+            "cargo",
+            "+nightly",
+            "run",
+            "build",
+            "-m",
+            str(crate / "Cargo.toml"),
+            "--target",
+            "wasm32-unknown-emscripten",
+            "-i",
+            f"python{ver.major}.{ver.minor}",
+            external=True,
+        )
+
+        with session.chdir(emscripten_dir):
+            session.run("node", "emscripten_runner.js", str(crate), external=True)

src/auditwheel/policy.rs

@@ -104,6 +104,7 @@ impl Policy {
                 Arch::X86 => "libc.musl-x86.so.1",
                 Arch::X86_64 => "libc.musl-x86_64.so.1",
                 Arch::S390X => "libc.musl-s390x.so.1",
+                _ => "",
             };
             if !new_soname.is_empty() {
                 self.lib_whitelist.insert(new_soname.to_string());

src/compile.rs

@@ -5,7 +5,9 @@ use fat_macho::FatWriter;
 use fs_err::{self as fs, File};
 use std::collections::HashMap;
 use std::env;
-use std::io::{BufReader, Read};
+use std::io::{BufReader, Read, Write};
+#[cfg(unix)]
+use std::os::unix::fs::PermissionsExt;
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
 use std::str;
@@ -14,6 +16,8 @@ use std::str;
 /// without `PYO3_NO_PYTHON` environment variable
 const PYO3_ABI3_NO_PYTHON_VERSION: (u64, u64, u64) = (0, 16, 4);
 
+const EMCC_WRAPPER: &str = include_str!("emcc_wrapper.py");
+
 /// Builds the rust crate into a native module (i.e. an .so or .dll) for a
 /// specific python version. Returns a mapping from crate type (e.g. cdylib)
 /// to artifact location.
@@ -195,6 +199,18 @@ fn compile_target(
             ];
             cargo_rustc.args.extend(mac_args);
         }
+    } else if target.is_emscripten() {
+        cargo_rustc.unstable_flags.push("build-std".to_string());
+        rust_flags
+            .get_or_insert_with(Default::default)
+            .push(" -C relocation-model=pic");
+        let emscripten_args = [
+            "-C".to_string(),
+            "link-arg=-sSIDE_MODULE=2".to_string(),
+            "-C".to_string(),
+            "link-arg=-sWASM_BIGINT".to_string(),
+        ];
+        cargo_rustc.args.extend(emscripten_args);
     }
 
     if context.strip {
@@ -244,6 +260,28 @@ fn compile_target(
         }
     }
 
+    if target.is_emscripten() {
+        // Workaround https://github.com/emscripten-core/emscripten/issues/17191
+        let cache_dir = dirs::cache_dir()
+            .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir"))
+            .join(env!("CARGO_PKG_NAME"));
+        fs::create_dir_all(&cache_dir)?;
+        let emcc_wrapper = cache_dir.join("emcc_wrapper.py");
+        let mut emcc_wrapper_file = fs::File::create(&emcc_wrapper)?;
+        emcc_wrapper_file.write_all(EMCC_WRAPPER.as_bytes())?;
+        #[cfg(unix)]
+        {
+            let metadata = emcc_wrapper_file.metadata()?;
+            let mut permissions = metadata.permissions();
+            permissions.set_mode(0o755);
+            emcc_wrapper_file.set_permissions(permissions)?;
+        }
+        build_command.env(
+            "CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER",
+            emcc_wrapper,
+        );
+    }
+
     build_command
         // We need to capture the json messages
         .stdout(Stdio::piped())
@@ -334,11 +372,17 @@ fn compile_target(
                 let crate_name = match package_in_metadata {
                     Some(package) => &package.name,
                     None => {
-                        // This is a spurious error I don't really understand
-                        println!(
-                            "⚠️  Warning: The package {} wasn't listed in `cargo metadata`",
-                            artifact.package_id
-                        );
+                        let package_id = &artifact.package_id;
+                        // Ignore the package if it's coming from Rust sysroot when compiling with `-Zbuild-std`
+                        if !package_id.repr.contains("rustup")
+                            && !package_id.repr.contains("rustlib")
+                        {
+                            // This is a spurious error I don't really understand
+                            println!(
+                                "⚠️  Warning: The package {} wasn't listed in `cargo metadata`",
+                                package_id
+                            );
+                        }
                         continue;
                     }
                 };

src/emcc_wrapper.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+
+
+def update_args(args):
+    # remove -lc. Not sure if it makes a difference but -lc doesn't belong here.
+    # https://github.com/emscripten-core/emscripten/issues/17191
+    for i in reversed(range(len(args))):
+        if args[i] == "c" and args[i - 1] == "-l":
+            del args[i - 1 : i + 1]
+
+    return args
+
+
+def main(args):
+    args = update_args(args)
+    return subprocess.call(["emcc"] + args)
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    sys.exit(main(args))

src/python_interpreter/config.rs

@@ -37,6 +37,11 @@ static WELLKNOWN_SYSCONFIG: Lazy<HashMap<Os, HashMap<Arch, Vec<InterpreterConfig
         let sysconfig_netbsd = serde_json::from_slice(include_bytes!("sysconfig-netbsd.json"))
             .expect("invalid sysconfig-netbsd.json");
         sysconfig.insert(Os::NetBsd, sysconfig_netbsd);
+        // Emscripten
+        let sysconfig_emscripten =
+            serde_json::from_slice(include_bytes!("sysconfig-emscripten.json"))
+                .expect("invalid sysconfig-emscripten.json");
+        sysconfig.insert(Os::Emscripten, sysconfig_emscripten);
         sysconfig
     });
 

src/python_interpreter/sysconfig-emscripten.json

@@ -0,0 +1,22 @@
+{
+  "wasm32": [
+    {
+      "major": 3,
+      "minor": 10,
+      "abiflags": "",
+      "interpreter": "cpython",
+      "ext_suffix": ".cpython-310-wasm32-emscripten.so",
+      "abi_tag": "310",
+      "pointer_width": 32
+    },
+    {
+      "major": 3,
+      "minor": 11,
+      "abiflags": "",
+      "interpreter": "cpython",
+      "ext_suffix": ".cpython-311-wasm32-emscripten.so",
+      "abi_tag": "311",
+      "pointer_width": 32
+    }
+  ]
+}