From 292fca0437b5cf0ec3190c4194ab327fe83fc82b Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 29 Aug 2022 17:25:24 +0200 Subject: [PATCH 1/3] gh-95853: Improve WASM build script - pre-build Emscripten ports and system libraries - check for broken EMSDK versions - use EMSDK's node for wasm32-emscripten - warn when PKG_CONFIG_PATH is set - add support level information --- ...2-08-29-17-25-13.gh-issue-95853.Ce17cT.rst | 2 + Tools/wasm/wasm_build.py | 202 ++++++++++++++++-- 2 files changed, 191 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst new file mode 100644 index 00000000000000..1cd1ce14fac08c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst @@ -0,0 +1,2 @@ +The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for +broken EMSDK versions, and warns about pkg-config env vars. diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 5ccf88cbc44fd0..b0f4790a36d277 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -25,6 +25,8 @@ import shutil import subprocess import sysconfig +import tempfile +import warnings # for Python 3.8 from typing import Any, Dict, Callable, Iterable, List, Optional, Union @@ -45,6 +47,11 @@ EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) # 3.1.16 has broken utime() EMSDK_MIN_VERSION = (3, 1, 17) +EMSDK_BROKEN_VERSION = { + (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", + (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393", + (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720", +} _MISSING = pathlib.PurePath("MISSING") # WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py" @@ -80,24 +87,28 @@ """ -def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.PurePath: - """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT +def get_emscripten_root( + emconfig: pathlib.Path = EM_CONFIG, +) -> Iterable[pathlib.PurePath]: + """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten" subdirectory with tools like "emconfigure". """ if not emconfig.exists(): - return _MISSING + return _MISSING, _MISSING with open(emconfig, encoding="utf-8") as f: code = f.read() # EM_CONFIG file is a Python snippet local: Dict[str, Any] = {} exec(code, globals(), local) - return pathlib.Path(local["EMSCRIPTEN_ROOT"]) + emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"]) + node_js = pathlib.Path(local["NODE_JS"]) + return emscripten_root, node_js -EMSCRIPTEN_ROOT = get_emscripten_root() +EMSCRIPTEN_ROOT, NODE_JS = get_emscripten_root() def read_python_version(configure: pathlib.Path = CONFIGURE) -> str: @@ -153,6 +164,9 @@ class Platform: make_wrapper: Optional[pathlib.PurePath] environ: dict check: Callable[[], None] + # used for build_emports() + ports: Optional[pathlib.PurePath] + cc: Optional[pathlib.PurePath] def getenv(self, profile: "BuildProfile") -> dict: return self.environ.copy() @@ -174,6 +188,8 @@ def _check_clean_src(): pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python", config_site=None, configure_wrapper=None, + ports=None, + cc=None, make_wrapper=None, environ={}, check=_check_clean_src, @@ -198,12 +214,24 @@ def _check_emscripten(): version = version[:-4] version_tuple = tuple(int(v) for v in version.split(".")) if version_tuple < EMSDK_MIN_VERSION: - raise MissingDependency( + raise ConditionError( os.fspath(version_txt), f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than " "minimum required version " f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.", ) + broken = EMSDK_BROKEN_VERSION.get(version_tuple) + if broken is not None: + raise ConditionError( + os.fspath(version_txt), + f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known ", + f"bugs, see {broken}.", + ) + if os.environ.get("PKG_CONFIG_PATH"): + warnings.warn( + "PKG_CONFIG_PATH is set and not empty. emconfigure overrides " + "this environment variable. Use EM_PKG_CONFIG_PATH instead." + ) _check_clean_src() @@ -212,11 +240,14 @@ def _check_emscripten(): pythonexe="python.js", config_site=WASMTOOLS / "config.site-wasm32-emscripten", configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure", + ports=EMSCRIPTEN_ROOT / "embuilder", + cc=EMSCRIPTEN_ROOT / "emcc", make_wrapper=EMSCRIPTEN_ROOT / "emmake", environ={ # workaround for https://github.com/emscripten-core/emscripten/issues/17635 "TZ": "UTC", "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None, + "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]], }, check=_check_emscripten, ) @@ -237,6 +268,8 @@ def _check_wasi(): pythonexe="python.wasm", config_site=WASMTOOLS / "config.site-wasm32-wasi", configure_wrapper=WASMTOOLS / "wasi-env", + ports=None, + cc=WASI_SDK_PATH / "bin" / "clang", make_wrapper=None, environ={ "WASI_SDK_PATH": WASI_SDK_PATH, @@ -246,6 +279,7 @@ def _check_wasi(): "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib " "--mapdir /::{srcdir} --" ), + "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], }, check=_check_wasi, ) @@ -280,6 +314,42 @@ def is_wasi(self) -> bool: cls = type(self) return self in {cls.wasm32_wasi, cls.wasm64_wasi} + def get_extra_paths(self) -> Iterable[pathlib.PurePath]: + """Host-specific os.environ["PATH"] entries + + Emscripten's Node version 14.x works well for wasm32-emscripten. + wasm64-emscripten requires more recent v8 version, e.g. node 16.x. + Attempt to use system's node command. + """ + cls = type(self) + if self == cls.wasm32_emscripten: + return [NODE_JS.parent] + elif self == cls.wasm64_emscripten: + # TODO: look for recent node + return [] + else: + return [] + + @property + def emport_args(self) -> List[str]: + """Host-specific port args (Emscripten)""" + cls = type(self) + if self is cls.wasm64_emscripten: + return ["-sMEMORY64=1"] + elif self is cls.wasm32_emscripten: + return ["-sMEMORY64=0"] + else: + return [] + + @property + def embuilder_args(self) -> List[str]: + """Host-specific embuilder args (Emscripten)""" + cls = type(self) + if self is cls.wasm64_emscripten: + return ["--wasm64"] + else: + return [] + class EmscriptenTarget(enum.Enum): """Emscripten-specific targets (--with-emscripten-target)""" @@ -294,10 +364,32 @@ def can_execute(self) -> bool: cls = type(self) return self not in {cls.browser, cls.browser_debug} + @property + def emport_args(self) -> List[str]: + """Target-specific port args""" + cls = type(self) + if self in {cls.browser_debug, cls.node_debug}: + # some libs come in debug and non-debug builds + return ["-O0"] + else: + return ["-O2"] + + +class SupportLevel(enum.Enum): + supported = "tier 3, supported" + working = "working, unsupported" + experimental = "experimental, may be broken" + broken = "broken / unavailable" + + def __bool__(self): + cls = type(self) + return self in {cls.supported, cls.working} + @dataclasses.dataclass class BuildProfile: name: str + support_level: SupportLevel host: Host target: Union[EmscriptenTarget, None] = None dynamic_linking: Union[bool, None] = None @@ -380,6 +472,12 @@ def getenv(self) -> dict: for key, value in platenv.items(): if value is None: env.pop(key, None) + elif key == "PATH": + # list of path items, prefix with extra paths + new_path: List[pathlib.PurePath] = [] + new_path.extend(self.host.get_extra_paths()) + new_path.extend(value) + env[key] = os.pathsep.join(os.fspath(p) for p in new_path) elif isinstance(value, str): env[key] = value.format( relbuilddir=self.builddir.relative_to(SRCDIR), @@ -390,12 +488,19 @@ def getenv(self) -> dict: env[key] = value return env - def _run_cmd(self, cmd: Iterable[str], args: Iterable[str]): + def _run_cmd( + self, + cmd: Iterable[str], + args: Iterable[str] = (), + cwd: Optional[pathlib.Path] = None, + ): cmd = list(cmd) cmd.extend(args) + if cwd is None: + cwd = self.builddir return subprocess.check_call( cmd, - cwd=os.fspath(self.builddir), + cwd=os.fspath(cwd), env=self.getenv(), ) @@ -443,10 +548,57 @@ def clean(self, all: bool = False): elif self.makefile.exists(): self.run_make("clean") + def build_emports(self, force: bool = False): + """Pre-build emscripten ports""" + platform = self.host.platform + if platform.ports is None or platform.cc is None: + raise ValueError("Need ports and CC command") + + embuilder_cmd = [os.fspath(platform.ports)] + embuilder_cmd.extend(self.host.embuilder_args) + if force: + embuilder_cmd.append("--force") + + ports_cmd = [os.fspath(platform.cc)] + ports_cmd.extend(self.host.emport_args) + if self.target: + ports_cmd.extend(self.target.emport_args) + + if self.dynamic_linking: + # trigger PIC build + ports_cmd.append("-sMAIN_MODULE") + embuilder_cmd.append("--pic") + if self.pthreads: + # trigger multi-threaded build + ports_cmd.append("-sUSE_PTHREADS") + # embuilder_cmd.append("--pthreads") + + # pre-build libbz2, libsqlite3, libz, and some system libs + ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) + embuilder_cmd.extend(["build", "bzip2", "sqlite3", "zlib"]) + + if not self.pthreads: + # Emscripten <= 3.1.20 has no option to build multi-threaded ports. + self._run_cmd(embuilder_cmd, cwd=SRCDIR) + + with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir: + tmppath = pathlib.Path(tmpdir) + main_c = tmppath / "main.c" + main_js = tmppath / "main.js" + with main_c.open("w") as f: + f.write("int main(void) { return 0; }\n") + args = [ + os.fspath(main_c), + "-o", + os.fspath(main_js), + ] + self._run_cmd(ports_cmd, args, cwd=tmppath) + # native build (build Python) BUILD = BuildProfile( "build", + support_level=SupportLevel.working, host=Host.build, ) @@ -455,43 +607,59 @@ def clean(self, all: bool = False): # wasm32-emscripten BuildProfile( "emscripten-browser", + support_level=SupportLevel.supported, host=Host.wasm32_emscripten, target=EmscriptenTarget.browser, dynamic_linking=True, ), BuildProfile( "emscripten-browser-debug", + support_level=SupportLevel.working, host=Host.wasm32_emscripten, target=EmscriptenTarget.browser_debug, dynamic_linking=True, ), BuildProfile( "emscripten-node-dl", + support_level=SupportLevel.supported, host=Host.wasm32_emscripten, target=EmscriptenTarget.node, dynamic_linking=True, ), BuildProfile( "emscripten-node-dl-debug", + support_level=SupportLevel.working, host=Host.wasm32_emscripten, target=EmscriptenTarget.node_debug, dynamic_linking=True, ), BuildProfile( "emscripten-node-pthreads", + support_level=SupportLevel.supported, host=Host.wasm32_emscripten, target=EmscriptenTarget.node, pthreads=True, ), BuildProfile( "emscripten-node-pthreads-debug", + support_level=SupportLevel.working, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node_debug, + pthreads=True, + ), + # Emscripten build with both pthreads and dynamic linking is crashing. + BuildProfile( + "emscripten-node-dl-pthreads-debug", + support_level=SupportLevel.broken, host=Host.wasm32_emscripten, target=EmscriptenTarget.node_debug, + dynamic_linking=True, pthreads=True, ), - # wasm64-emscripten (currently not working) + # wasm64-emscripten (requires unreleased Emscripten >= 3.1.21) BuildProfile( "wasm64-emscripten-node-debug", + support_level=SupportLevel.experimental, host=Host.wasm64_emscripten, target=EmscriptenTarget.node_debug, # MEMORY64 is not compatible with dynamic linking @@ -501,6 +669,7 @@ def clean(self, all: bool = False): # wasm32-wasi BuildProfile( "wasi", + support_level=SupportLevel.supported, host=Host.wasm32_wasi, # skip sysconfig test_srcdir testopts="-i '*.test_srcdir' -j2", @@ -508,6 +677,7 @@ def clean(self, all: bool = False): # no SDK available yet # BuildProfile( # "wasm64-wasi", + # support_level=SupportLevel.broken, # host=Host.wasm64_wasi, # ), ] @@ -523,15 +693,17 @@ def clean(self, all: bool = False): "--clean", "-c", help="Clean build directories first", action="store_true" ) -platforms = list(PROFILES) + ["cleanall"] +# Don't list broken and experimental variants in help +platforms_choices = list(p.name for p in _profiles) + ["cleanall"] +platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] parser.add_argument( "platform", metavar="PLATFORM", - help=f"Build platform: {', '.join(platforms)}", - choices=platforms, + help=f"Build platform: {', '.join(platforms_help)}", + choices=platforms_choices, ) -ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall"] +ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall", "emports"] parser.add_argument( "op", metavar="OP", @@ -572,6 +744,8 @@ def main(): builder.clean(all=False) if args.op == "compile": + if builder.host.is_emscripten: + builder.build_emports() builder.run_build(force_configure=True) else: if not builder.makefile.exists(): @@ -586,6 +760,8 @@ def main(): builder.clean(all=False) elif args.op == "cleanall": builder.clean(all=True) + elif args.op == "emports": + builder.build_emports(force=args.clean) else: raise ValueError(args.op) From 4a14001685fb9d788439d3f6aa25cb473a160658 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 30 Aug 2022 07:08:24 +0200 Subject: [PATCH 2/3] Mind the full stop Co-authored-by: Brett Cannon --- Tools/wasm/wasm_build.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index b0f4790a36d277..393460aed2e0a7 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -90,7 +90,7 @@ def get_emscripten_root( emconfig: pathlib.Path = EM_CONFIG, ) -> Iterable[pathlib.PurePath]: - """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS + """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS. The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten" @@ -164,7 +164,7 @@ class Platform: make_wrapper: Optional[pathlib.PurePath] environ: dict check: Callable[[], None] - # used for build_emports() + # Used for build_emports(). ports: Optional[pathlib.PurePath] cc: Optional[pathlib.PurePath] @@ -224,8 +224,8 @@ def _check_emscripten(): if broken is not None: raise ConditionError( os.fspath(version_txt), - f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known ", - f"bugs, see {broken}.", + (f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known " + f"bugs, see {broken}."), ) if os.environ.get("PKG_CONFIG_PATH"): warnings.warn( @@ -315,7 +315,7 @@ def is_wasi(self) -> bool: return self in {cls.wasm32_wasi, cls.wasm64_wasi} def get_extra_paths(self) -> Iterable[pathlib.PurePath]: - """Host-specific os.environ["PATH"] entries + """Host-specific os.environ["PATH"] entries. Emscripten's Node version 14.x works well for wasm32-emscripten. wasm64-emscripten requires more recent v8 version, e.g. node 16.x. @@ -332,7 +332,7 @@ def get_extra_paths(self) -> Iterable[pathlib.PurePath]: @property def emport_args(self) -> List[str]: - """Host-specific port args (Emscripten)""" + """Host-specific port args (Emscripten).""" cls = type(self) if self is cls.wasm64_emscripten: return ["-sMEMORY64=1"] @@ -343,7 +343,7 @@ def emport_args(self) -> List[str]: @property def embuilder_args(self) -> List[str]: - """Host-specific embuilder args (Emscripten)""" + """Host-specific embuilder args (Emscripten).""" cls = type(self) if self is cls.wasm64_emscripten: return ["--wasm64"] @@ -366,7 +366,7 @@ def can_execute(self) -> bool: @property def emport_args(self) -> List[str]: - """Target-specific port args""" + """Target-specific port args.""" cls = type(self) if self in {cls.browser_debug, cls.node_debug}: # some libs come in debug and non-debug builds @@ -549,7 +549,7 @@ def clean(self, all: bool = False): self.run_make("clean") def build_emports(self, force: bool = False): - """Pre-build emscripten ports""" + """Pre-build emscripten ports.""" platform = self.host.platform if platform.ports is None or platform.cc is None: raise ValueError("Need ports and CC command") @@ -565,15 +565,15 @@ def build_emports(self, force: bool = False): ports_cmd.extend(self.target.emport_args) if self.dynamic_linking: - # trigger PIC build + # Trigger PIC build. ports_cmd.append("-sMAIN_MODULE") embuilder_cmd.append("--pic") if self.pthreads: - # trigger multi-threaded build + # Trigger multi-threaded build. ports_cmd.append("-sUSE_PTHREADS") # embuilder_cmd.append("--pthreads") - # pre-build libbz2, libsqlite3, libz, and some system libs + # Pre-build libbz2, libsqlite3, libz, and some system libs. ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) embuilder_cmd.extend(["build", "bzip2", "sqlite3", "zlib"]) From f4be1636a32dcb46aa75a5b7b847708f4bc5c5b1 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 30 Aug 2022 07:22:18 +0200 Subject: [PATCH 3/3] Rename func to parse_emconfig, comment, black --- Tools/wasm/wasm_build.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 393460aed2e0a7..9054370e5e2fdc 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -29,7 +29,7 @@ import warnings # for Python 3.8 -from typing import Any, Dict, Callable, Iterable, List, Optional, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() WASMTOOLS = SRCDIR / "Tools" / "wasm" @@ -87,9 +87,9 @@ """ -def get_emscripten_root( +def parse_emconfig( emconfig: pathlib.Path = EM_CONFIG, -) -> Iterable[pathlib.PurePath]: +) -> Tuple[pathlib.PurePath, pathlib.PurePath]: """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS. The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" @@ -108,7 +108,7 @@ def get_emscripten_root( return emscripten_root, node_js -EMSCRIPTEN_ROOT, NODE_JS = get_emscripten_root() +EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig() def read_python_version(configure: pathlib.Path = CONFIGURE) -> str: @@ -224,8 +224,10 @@ def _check_emscripten(): if broken is not None: raise ConditionError( os.fspath(version_txt), - (f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known " - f"bugs, see {broken}."), + ( + f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known " + f"bugs, see {broken}." + ), ) if os.environ.get("PKG_CONFIG_PATH"): warnings.warn( @@ -571,6 +573,7 @@ def build_emports(self, force: bool = False): if self.pthreads: # Trigger multi-threaded build. ports_cmd.append("-sUSE_PTHREADS") + # https://github.com/emscripten-core/emscripten/pull/17729 # embuilder_cmd.append("--pthreads") # Pre-build libbz2, libsqlite3, libz, and some system libs.