From 8ae236393cf53b17cb72b729e7909b2f29b7336a Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 11 Sep 2023 18:48:38 +0200 Subject: [PATCH] bigint pre-release test --- config | 14 +- scripts/cpython-build-emsdk-prebuilt.sh | 45 ++++--- scripts/cpython-build-emsdk.sh | 33 +---- scripts/cpython-build-host-deps.sh | 6 +- scripts/cpython-fetch.sh | 15 +-- scripts/emsdk-fetch.sh | 55 ++++++-- support/__EMSCRIPTEN__.embed/cpython3.11.diff | 127 +++++++++++++++++- 7 files changed, 215 insertions(+), 80 deletions(-) diff --git a/config b/config index 1fb3f7c..f883ea0 100644 --- a/config +++ b/config @@ -20,7 +20,7 @@ export PYTHONPYCACHEPREFIX=${PYTHONPYCACHEPREFIX:-$(realpath ${SDKROOT}/build/py export HOME=${SDKROOT} export HPY=${HOST_PREFIX}/bin/python${PYBUILD} -export PIP=${HOST_PREFIX}/bin/pip${PYBUILD} +export HPIP=${HOST_PREFIX}/bin/pip${PYBUILD} # cpython build opts export CPOPTS="-Os -g0 -fPIC" @@ -49,14 +49,20 @@ fi EMFLAVOUR=${EMFLAVOUR:-tot} -# no extra wasm features -export WASMFLAVOUR=mvp +# base wasm features pure is "mvp" +export WASM_FLAVOUR=bi + +# extra features. Safari 14 is expected to have BIGINT, ignore the emsdk warnings. +export WASM_EXTRA="-sWASM_BIGINT -sPOLYFILL -sMIN_SAFARI_VERSION=140000" + +# tell to not normalize modules. +export WASM_PURE=false #temp fix for oom on CI (Error: Process completed with exit code 143.) export EMSDK_NUM_CORES=1 -export PYDK_PYTHON_HOST_PLATFORM=wasm32-$WASMFLAVOUR-emscripten +export PYDK_PYTHON_HOST_PLATFORM=wasm32-${WASM_FLAVOUR}-emscripten if echo $LD_LIBRARY_PATH |grep -q ${HOST_PREFIX}/lib then diff --git a/scripts/cpython-build-emsdk-prebuilt.sh b/scripts/cpython-build-emsdk-prebuilt.sh index cc526ad..441c3ea 100755 --- a/scripts/cpython-build-emsdk-prebuilt.sh +++ b/scripts/cpython-build-emsdk-prebuilt.sh @@ -2,21 +2,20 @@ . ${CONFIG:-config} -CYTHON_REL=${CYTHON_REL:-3.0.0} +CYTHON_REL=${CYTHON_REL:-3.0.1} CYTHON_WHL=${CYTHON:-Cython-${CYTHON_REL}-py2.py3-none-any.whl} +# all needed for PEP722/723 PACKAGING="pip build wheel pyparsing packaging installer" +$HPIP install --upgrade $PACKAGING -$HPY -m pip install --upgrade $PACKAGING +# setuptools for HPy/static +$HPIP install --upgrade setuptools -# only for the simulator -$HPY -m pip install --upgrade aioconsole +# aioconsole only for the simulator +$HPIP install --upgrade aioconsole -pushd build - wget -q -c https://github.com/cython/cython/releases/download/${CYTHON_REL}/${CYTHON_WHL} - $HPY install --upgrade $CYTHON_WHL -popd PIP="${SDKROOT}/python3-wasm -m pip" @@ -28,20 +27,30 @@ echo " # support package build/install $HPY -m pip install --upgrade $PACKAGING + $PIP install --upgrade $PACKAGING +# setuptools for HPy/static +$PIP install --upgrade setuptools + + + # make wheels # /opt/python-wasm-sdk/python3-wasm setup.py bdist_wheel -# cython -$HPY -m pip install build/$CYTHON_WHL +# cython get the latest release on gh +pushd build + wget -q -c https://github.com/cython/cython/releases/download/${CYTHON_REL}/${CYTHON_WHL} + $HPIP install --upgrade $CYTHON_WHL +popd + $PIP install build/$CYTHON_WHL # some we want to be certain to have in all minimal rootfs mkdir -p prebuilt/emsdk/common/site-packages/ -for pkg in pyparsing packaging installer +for pkg in pyparsing packaging installer pkg_resources do if [ -d prebuilt/emsdk/${PYBUILD}/site-packages/$pkg ] then @@ -49,10 +58,16 @@ do $pkg already set to prebuilt " else - $PIP install $pkg - cp -rf $PREFIX/lib/python${PYBUILD}/site-packages/${pkg} prebuilt/emsdk/common/site-packages/ - # skip the distinfo to save space - #cp -rf $PREFIX/lib/python${PYBUILD}/site-packages/${pkg}-* prebuilt/emsdk/common/site-packages/ + if [ -d ${HOST_PREFIX}/lib/python${PYBUILD}/site-packages/${pkg} ] + then + cp -rf ${HOST_PREFIX}/lib/python${PYBUILD}/site-packages/${pkg} prebuilt/emsdk/common/site-packages/ + # skip the distinfo to save some space + #cp -rf ${HOST_PREFIX}/lib/python${PYBUILD}/site-packages/${pkg}-* prebuilt/emsdk/common/site-packages/ + + else + cp -rf ${ROOT}/.local/lib/python${PYBUILD}/site-packages/{$pkg} prebuilt/emsdk/common/site-packages/ + # skip the distinfo to save some space + fi fi done diff --git a/scripts/cpython-build-emsdk.sh b/scripts/cpython-build-emsdk.sh index d8441e8..2da773b 100755 --- a/scripts/cpython-build-emsdk.sh +++ b/scripts/cpython-build-emsdk.sh @@ -165,14 +165,15 @@ END # --with-libs='-lz -lffi' \ - - # MODULE__HASHLIB_LDFLAGS=-L/usr/lib -lcrypto pushd $ROOT/src/cpython${PYBUILD} + # fix double linking sed -i 's| -lcrypto||g' Makefile.pre.in + # please let compiler/user decide what to do with wasm CPU. sed -i 's|-sWASM_BIGINT||g' configure sed -i 's|-sWASM_BIGINT||g' configure.ac popd + PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig" CONFIG_SITE=$ROOT/src/cpython${PYBUILD}/Tools/wasm/config.site-wasm32-pydk \ emconfigure $ROOT/src/cpython${PYBUILD}/configure -C --with-emscripten-target=browser \ --cache-file=${PYTHONPYCACHEPREFIX}/config.cache \ @@ -187,31 +188,6 @@ END #echo "#define HAVE_NCURSES_H" >> pyconfig.h - if echo $PYBUILD|grep -q 3.10 - then - cat > Modules/Setup.local < Modules/Setup.local < ${PYTHONPYCACHEPREFIX}/.nanorc <&2 # install and update critical packages. -$PIP install --upgrade pip +$HPIP install --upgrade pip # 3.12 and git deprecated setuptools bundling. -$PIP install --upgrade setuptools +$HPIP install --upgrade setuptools diff --git a/scripts/cpython-fetch.sh b/scripts/cpython-fetch.sh index 2a00cb2..6bdc45e 100755 --- a/scripts/cpython-fetch.sh +++ b/scripts/cpython-fetch.sh @@ -17,6 +17,7 @@ PYPATCH=true [ -f $HPY ] || REBUILD=true + if echo $PYBUILD |grep -q 13$ then if [ -d cpython${PYBUILD} ] @@ -41,6 +42,7 @@ then fi fi + if echo $PYBUILD |grep -q 12$ then if [ -d cpython${PYBUILD} ] @@ -73,17 +75,6 @@ then export REBUILD=true fi -if echo $PYBUILD | grep -q 10$ -then - wget -q -c https://www.python.org/ftp/python/3.10.6/Python-3.10.6.tar.xz - tar xf Python-3.10.6.tar.xz - - ln -s Python-3.10.6 cpython${PYBUILD} - - NOPATCH=true - export REBUILD=true -fi - popd @@ -102,7 +93,7 @@ fi if $NOPATCH then - echo -n + echo "finally there" else # do some patching for 3.11+ to allow more shared libs pushd src/cpython${PYBUILD} 2>&1 >/dev/null diff --git a/scripts/emsdk-fetch.sh b/scripts/emsdk-fetch.sh index 097c15d..f8af438 100755 --- a/scripts/emsdk-fetch.sh +++ b/scripts/emsdk-fetch.sh @@ -173,16 +173,22 @@ SHARED="" IS_SHARED=false PY_MODULE=false MVP=\${MVP:true} +WASM_PURE=\${WASM_PURE:true} if \$MVP then # -mcpu=generic would activate those https://reviews.llvm.org/D125728 # https://github.com/emscripten-core/emscripten/pull/17689 - CPU="-sWASM_BIGINT=0 -sMIN_SAFARI_VERSION=120000 -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-mutable-globals -m32" + # CPU="-sWASM_BIGINT=0 -sMIN_SAFARI_VERSION=140000 -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-mutable-globals -m32" + # go hybrid + CPU="-sMIN_SAFARI_VERSION=140000 -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-mutable-globals -m32" else CPU="-mcpu=bleeding-edge -m32" fi +# quick hack until 3.1.47 +CPU="$WASM_EXTRA \$CPU" + LINKING=\${LINKING:-false} @@ -243,6 +249,16 @@ for arg do continue fi + if [ "\$arg" = "-c" ] + then + CPU_EXTRA=\$WASM_EXTRA + fi + + if [ "\$arg" = "-o" ] + then + CPU_EXTRA=\$WASM_EXTRA + fi + if [ "\$arg" = "-fallow-argument-mismatch" ] then continue @@ -320,17 +336,22 @@ done if \$IS_SHARED then - $EMSDK_PYTHON -E \$0.py \$SHARED $CPU $COPTS $LDFLAGS -sSIDE_MODULE -gsource-map --source-map-base / "\$@" \$COMMON + # always pass CPU opts when linking + $EMSDK_PYTHON -E \$0.py \$SHARED \$CPU $COPTS $LDFLAGS -sSIDE_MODULE -gsource-map --source-map-base / "\$@" \$COMMON if \$MVP then - SOTMP=\$(mktemp).so - mv \$SHARED_TARGET \$SOTMP - $SDKROOT/emsdk/upstream/bin/wasm-emscripten-finalize -mvp \$SOTMP -o \$SHARED_TARGET - [ -f \$SHARED_TARGET.map ] && rm \$SHARED_TARGET.map - rm \$SOTMP + if \$WASM_PURE + then + SOTMP=\$(mktemp).so + mv \$SHARED_TARGET \$SOTMP + $SDKROOT/emsdk/upstream/bin/wasm-emscripten-finalize -mvp \$SOTMP -o \$SHARED_TARGET + [ -f \$SHARED_TARGET.map ] && rm \$SHARED_TARGET.map + rm \$SOTMP + fi fi else - $EMSDK_PYTHON -E \$0.py $CPU -fpic \$COPTS \$CPPFLAGS -DBUILD_STATIC "\$@" \$COMMON + # pass CPU opts only when -c/-o but always PIC + $EMSDK_PYTHON -E \$0.py \$CPU_EXTRA -fpic \$COPTS \$CPPFLAGS -DBUILD_STATIC "\$@" \$COMMON fi #else # unset _EMCC_CCACHE @@ -339,7 +360,23 @@ fi END - cat emsdk/upstream/emscripten/emcc > emsdk/upstream/emscripten/em++ + rm emsdk/upstream/emscripten/em++ + if ln emsdk/upstream/emscripten/emcc emsdk/upstream/emscripten/em++ + then + # cmake usually wants cc + ln emsdk/upstream/emscripten/emcc emsdk/upstream/emscripten/cc + ln emsdk/upstream/emscripten/emcc.py emsdk/upstream/emscripten/cc.py + else + echo " + + ============ hard link not supported ============== + + + " + cat emsdk/upstream/emscripten/emcc > emsdk/upstream/emscripten/em++ + cat emsdk/upstream/emscripten/emcc > emsdk/upstream/emscripten/cc + cat emsdk/upstream/emscripten/emcc.py > emsdk/upstream/emscripten/cc.py + fi cat > emsdk/upstream/emscripten/emar <tok->interactive_src_start. + * If not, we're parsing from a string, which means that the whole source + * is stored in p->tok->str. */ +- assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp == stdin); ++ assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL); + + char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str; + if (cur_line == NULL) { +--- Python-3.11.5/Programs/python.c 2023-08-24 14:09:18.000000000 +0200 ++++ Python-3.11.5-wasm/Programs/python.c 2023-09-09 22:33:10.196291366 +0200 +@@ -2,16 +2,72 @@ + + #include "Python.h" + +-#ifdef MS_WINDOWS +-int +-wmain(int argc, wchar_t **argv) ++#if __PYDK__ ++#include "pycore_call.h" // _PyObject_CallNoArgs() ++#include "pycore_initconfig.h" // _PyArgv ++#include "pycore_interp.h" // _PyInterpreterState.sysdict ++#include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() ++#include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() ++#include "pycore_pystate.h" // _PyInterpreterState_GET() ++ ++static PyStatus ++pymain_init(const _PyArgv *args) ++{ ++ PyStatus status; ++ ++ status = _PyRuntime_Initialize(); ++ if (_PyStatus_EXCEPTION(status)) { ++ return status; ++ } ++ ++ PyPreConfig preconfig; ++ PyPreConfig_InitPythonConfig(&preconfig); ++ ++ status = _Py_PreInitializeFromPyArgv(&preconfig, args); ++ if (_PyStatus_EXCEPTION(status)) { ++ return status; ++ } ++ ++ PyConfig config; ++ PyConfig_InitPythonConfig(&config); ++ ++ if (args->use_bytes_argv) { ++ status = PyConfig_SetBytesArgv(&config, args->argc, args->bytes_argv); ++ } ++ else { ++ status = PyConfig_SetArgv(&config, args->argc, args->wchar_argv); ++ } ++ if (_PyStatus_EXCEPTION(status)) { ++ goto done; ++ } ++ ++ status = Py_InitializeFromConfig(&config); ++ if (_PyStatus_EXCEPTION(status)) { ++ goto done; ++ } ++ status = _PyStatus_OK(); ++ ++done: ++ PyConfig_Clear(&config); ++ return status; ++} ++ ++static void ++pymain_free(void) + { +- return Py_Main(argc, argv); ++ _PyImport_Fini2(); ++ _PyPathConfig_ClearGlobal(); ++ _Py_ClearStandardStreamEncoding(); ++ _Py_ClearArgcArgv(); ++ _PyRuntime_Finalize(); + } ++ ++#include "/opt/python-wasm-sdk/support/__EMSCRIPTEN__.c" + #else + int + main(int argc, char **argv) + { ++ + return Py_BytesMain(argc, argv); + } +-#endif ++#endif //#if __PYDK__ +--- Python-3.11.5/Python/sysmodule.c 2023-08-24 14:09:18.000000000 +0200 ++++ Python-3.11.5-wasm/Python/sysmodule.c 2023-09-09 22:33:10.196291366 +0200 @@ -49,7 +49,7 @@ extern const char *PyWin_DLLVersionString; #endif @@ -22,7 +135,7 @@ #include #endif -@@ -2748,7 +2748,7 @@ +@@ -2750,7 +2750,7 @@ return NULL; } @@ -31,7 +144,7 @@ PyDoc_STRVAR(emscripten_info__doc__, "sys._emscripten_info\n\ -@@ -2988,7 +2988,7 @@ +@@ -2990,7 +2990,7 @@ } } @@ -40,7 +153,7 @@ if (EmscriptenInfoType == NULL) { EmscriptenInfoType = PyStructSequence_NewType(&emscripten_info_desc); if (EmscriptenInfoType == NULL) { -@@ -3243,7 +3243,7 @@ +@@ -3251,7 +3251,7 @@ #endif _PyStructSequence_FiniType(&Hash_InfoType); _PyStructSequence_FiniType(&AsyncGenHooksType);