gh-95853: Add script to automate WASM build (GH-95828)

Lib/test/test_unicode_file_functions.py

@@ -6,6 +6,7 @@
 import warnings
 from unicodedata import normalize
 from test.support import os_helper
+from test import support
 
 
 filenames = [
@@ -123,6 +124,10 @@ def test_open(self):
     # NFKD in Python is useless, because darwin will normalize it later and so
     # open(), os.stat(), etc. don't raise any exception.
     @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
+    @unittest.skipIf(
+        support.is_emscripten or support.is_wasi,
+        "test fails on Emscripten/WASI when host platform is macOS."
+    )
     def test_normalize(self):
         files = set(self.files)
         others = set()

Lib/test/test_warnings/__init__.py

@@ -487,7 +487,14 @@ def test_warn_explicit_non_ascii_filename(self):
                 module=self.module) as w:
             self.module.resetwarnings()
             self.module.filterwarnings("always", category=UserWarning)
-            for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"):
+            filenames = ["nonascii\xe9\u20ac"]
+            if not support.is_emscripten:
+                # JavaScript does not like surrogates.
+                # Invalid UTF-8 leading byte 0x80 encountered when
+                # deserializing a UTF-8 string in wasm memory to a JS
+                # string!
+                filenames.append("surrogate\udc80")
+            for filename in filenames:
                 try:
                     os.fsencode(filename)
                 except UnicodeEncodeError:

Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst

@@ -0,0 +1,2 @@
+The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and
+test steps for building CPython on WebAssembly platforms.

Python/sysmodule.c

@@ -2789,14 +2789,18 @@ EM_JS(char *, _Py_emscripten_runtime, (void), {
     if (typeof navigator == 'object') {
         info = navigator.userAgent;
     } else if (typeof process == 'object') {
-        info = "Node.js ".concat(process.version)
+        info = "Node.js ".concat(process.version);
     } else {
-        info = "UNKNOWN"
+        info = "UNKNOWN";
     }
     var len = lengthBytesUTF8(info) + 1;
     var res = _malloc(len);
-    stringToUTF8(info, res, len);
+    if (res) stringToUTF8(info, res, len);
+#if __wasm64__
+    return BigInt(res);
+#else
     return res;
+#endif
 });
 
 static PyObject *

Tools/wasm/README.md

@@ -35,7 +35,13 @@ docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay.
 
 ### Compile a build Python interpreter
 
-From within the container, run the following commands:
+From within the container, run the following command:
+
+```shell
+./Tools/wasm/wasm_build.py build
+```
+
+The command is roughly equivalent to:
 
 ```shell
 mkdir -p builddir/build
@@ -45,13 +51,13 @@ make -j$(nproc)
 popd
 ```
 
-### Fetch and build additional emscripten ports
+### Cross-compile to wasm32-emscripten for browser
 
 ```shell
-embuilder build zlib bzip2
+./Tools/wasm/wasm_build.py emscripten-browser
 ```
 
-### Cross compile to wasm32-emscripten for browser
+The command is roughly equivalent to:
 
 ```shell
 mkdir -p builddir/emscripten-browser
@@ -85,22 +91,29 @@ and header files with debug builds.
 ### Cross compile to wasm32-emscripten for node
 
 ```shell
-mkdir -p builddir/emscripten-node
-pushd builddir/emscripten-node
+./Tools/wasm/wasm_build.py emscripten-browser-dl
+```
+
+The command is roughly equivalent to:
+
+```shell
+mkdir -p builddir/emscripten-node-dl
+pushd builddir/emscripten-node-dl
 
 CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
   emconfigure ../../configure -C \
     --host=wasm32-unknown-emscripten \
     --build=$(../../config.guess) \
     --with-emscripten-target=node \
+    --enable-wasm-dynamic-linking \
     --with-build-python=$(pwd)/../build/python
 
 emmake make -j$(nproc)
 popd
 ```
 
 ```shell
-node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node/python.js
+node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js
 ```
 
 (``--experimental-wasm-bigint`` is not needed with recent NodeJS versions)
@@ -199,6 +212,15 @@ Node builds use ``NODERAWFS``.
 - Node RawFS allows direct access to the host file system without need to
   perform ``FS.mount()`` call.
 
+## wasm64-emscripten
+
+- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``.
+- ``EM_JS`` functions must return ``BigInt()``.
+- ``Py_BuildValue()`` format strings must match size of types. Confusing 32
+  and 64 bits types leads to memory corruption, see
+  [gh-95876](https://github.com/python/cpython/issues/95876) and
+  [gh-95878](https://github.com/python/cpython/issues/95878).
+
 # Hosting Python WASM builds
 
 The simple REPL terminal uses SharedArrayBuffer. For security reasons
@@ -234,6 +256,12 @@ The script ``wasi-env`` sets necessary compiler and linker flags as well as
 ``pkg-config`` overrides. The script assumes that WASI-SDK is installed in
 ``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``.
 
+```shell
+./Tools/wasm/wasm_build.py wasi
+```
+
+The command is roughly equivalent to:
+
 ```shell
 mkdir -p builddir/wasi
 pushd builddir/wasi

Tools/wasm/wasi-env

@@ -72,4 +72,5 @@ export CFLAGS LDFLAGS
 export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR
 export PATH
 
-exec "$@"
+# no exec, it makes arvg[0] path absolute.
+"$@"