diff --git a/master/custom/builders.py b/master/custom/builders.py index 2855bb748..ef84ebba1 100644 --- a/master/custom/builders.py +++ b/master/custom/builders.py @@ -36,6 +36,9 @@ MacOSArmWithBrewBuild, WindowsARM64Build, WindowsARM64ReleaseBuild, + Wasm32EmscriptenNodeBuild, + Wasm32EmscriptenBrowserBuild, + Wasm32WASIBuild, ) STABLE = "stable" @@ -49,7 +52,7 @@ def get_builders(settings): # Override with a default simple worker if we are using local workers if settings.use_local_worker: - return [("Test Builder", "local-worker", UnixBuild, STABLE)] + return [("Test Builder", "local-worker", UnixBuild, STABLE, NO_TIER)] return [ # -- Stable builders -- @@ -207,6 +210,11 @@ def get_builders(settings): ("ARM64 Windows Non-Debug", "linaro-win-arm64", WindowsARM64ReleaseBuild, STABLE, NO_TIER), ("ARM64 Windows Azure", "linaro2-win-arm64", WindowsARM64Build, UNSTABLE, NO_TIER), ("ARM64 Windows Non-Debug Azure", "linaro2-win-arm64", WindowsARM64ReleaseBuild, UNSTABLE, NO_TIER), + + # WebAssembly + ("wasm32-emscripten node (threaded)", "bcannon-wasm", Wasm32EmscriptenNodeBuild, UNSTABLE, NO_TIER), + ("wasm32-emscripten browser (dynamic linking)", "bcannon-wasm", Wasm32EmscriptenBrowserBuild, UNSTABLE, NO_TIER), + ("wasm32-wasi", "bcannon-wasm", Wasm32WASIBuild, UNSTABLE, NO_TIER), ] diff --git a/master/custom/factories.py b/master/custom/factories.py index 25cedeb91..15d0b8e90 100644 --- a/master/custom/factories.py +++ b/master/custom/factories.py @@ -1,6 +1,13 @@ import os.path from buildbot.process import factory -from buildbot.steps.shell import Configure, Compile, ShellCommand +from buildbot.steps.shell import ( + Configure, + Compile, + ShellCommand, + SetPropertyFromCommand, +) + +from buildbot.plugins import util from .steps import ( Test, @@ -577,3 +584,253 @@ class WindowsARM64ReleaseBuild(WindowsARM64Build): testFlags = WindowsARM64Build.testFlags + ["+d"] # keep default cleanFlags, both configurations get cleaned factory_tags = ["win-arm64", "nondebug"] + +############################################################################## +############################## WASM BUILDS ################################# +############################################################################## + + +class UnixCrossBuild(UnixBuild): + configureFlags = [ + "--with-pydebug", + "--with-build-python=../build/python" + ] + extra_configure_flags = [] + host_configure_cmd = ["../../configure"] + host = None + host_make_cmd = ["make"] + can_execute_python = True + + def setup(self, parallel, branch, test_with_PTY=False, **kwargs): + assert self.host is not None, "Must set self.host on cross builds" + + out_of_tree_dir = "build_oot" + oot_dir_path = os.path.join("build", out_of_tree_dir) + oot_build_path = os.path.join(oot_dir_path, "build") + oot_host_path = os.path.join(oot_dir_path, "host") + + self.addStep( + SetPropertyFromCommand( + name="Gather build triple from worker", + description="Get the build triple config.guess", + command="./config.guess", + property="build_triple", + warnOnFailure=True, + ) + ) + + # Create out of tree directory for "build", the platform we are + # currently running on + self.addStep( + ShellCommand( + name="mkdir build out-of-tree directory", + description="Create build out-of-tree directory", + command=["mkdir", "-p", oot_build_path], + warnOnFailure=True, + ) + ) + # Create directory for "host", the platform we want to compile *for* + self.addStep( + ShellCommand( + name="mkdir host out-of-tree directory", + description="Create host out-of-tree directory", + command=["mkdir", "-p", oot_host_path], + warnOnFailure=True, + ) + ) + + # First, we build the "build" Python, which we need to cross compile + # the "host" Python + self.addStep( + Configure( + name="Configure build Python", + command=["../../configure"], + workdir=oot_build_path + ) + ) + if parallel: + compile = ["make", parallel] + else: + compile = ["make"] + + self.addStep( + Compile( + name="Compile build Python", + command=compile, + workdir=oot_build_path + ) + ) + + # Now that we have a "build" architecture Python, we can use that + # to build a "host" (also known as the target we are cross compiling) + # to + configure_cmd = self.host_configure_cmd + ["--prefix", "$(PWD)/target/host"] + configure_cmd += self.configureFlags + self.extra_configure_flags + configure_cmd += [util.Interpolate("--build=%(prop:build_triple)s")] + configure_cmd += [f"--host={self.host}"] + self.addStep( + Configure( + name="Configure host Python", + command=configure_cmd, + env=self.compile_environ, + workdir=oot_host_path + ) + ) + + testopts = self.testFlags + if "-R" not in self.testFlags: + testopts += " --junit-xml test-results.xml" + if parallel: + testopts = testopts + " " + parallel + if "-j" not in testopts: + testopts = "-j2 " + testopts + + # Timeout for the buildworker process + self.test_timeout = self.test_timeout or TEST_TIMEOUT + # Timeout for faulthandler + faulthandler_timeout = self.test_timeout - 5 * 60 + + test = [ + "make", + "buildbottest", + "TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}", + "TESTPYTHONOPTS=" + self.interpreterFlags, + "TESTTIMEOUT=" + str(faulthandler_timeout), + ] + + if parallel: + compile = self.host_make_cmd + [parallel, self.makeTarget] + else: + compile = self.host_make_cmd + [self.makeTarget] + self.addStep( + Compile( + name="Compile host Python", + command=compile, + env=self.compile_environ, + workdir=oot_host_path, + ) + ) + if self.can_execute_python: + self.addStep( + ShellCommand( + name="pythoninfo", + description="pythoninfo", + command=["make", "pythoninfo"], + warnOnFailure=True, + env=self.test_environ, + workdir=oot_host_path, + ) + ) + self.addStep( + Test( + command=test, + timeout=self.test_timeout, + usePTY=test_with_PTY, + env=self.test_environ, + workdir=oot_host_path, + ) + ) + if branch not in ("3",) and "-R" not in self.testFlags: + filename = os.path.join(oot_host_path, "test-results.xml") + self.addStep(UploadTestResults(branch, filename=filename)) + self.addStep( + Clean( + name="Clean build Python", + workdir=oot_build_path, + ) + ) + self.addStep( + Clean( + name="Clean host Python", + workdir=oot_host_path, + ) + ) + + +class Wasm32EmscriptenBuild(UnixCrossBuild): + """wasm32-emscripten builder + + * Emscripten SDK >= 3.1.12 must be installed + * ccache must be installed + * Emscripten PATHs must be pre-pended to PATH + * ``which node`` must be equal $EMSDK_NODE + """ + factory_tags = ["wasm", "emscripten"] + compile_environ = { + "CONFIG_SITE": "../../Tools/wasm/config.site-wasm32-emscripten", + "EM_COMPILER_WRAPPER": "ccache", + } + + host = "wasm32-unknown-emscripten" + host_configure_cmd = ["emconfigure", "../../configure"] + host_make_cmd = ["emmake", "make"] + + +class Wasm32EmscriptenNodeBuild(Wasm32EmscriptenBuild): + buildersuffix = ".emscripten-node" + extra_configure_flags = [ + "--with-emscripten-target=node", + "--disable-wasm-dynamic-linking", + "--enable-wasm-pthreads", + ] + + +class Wasm32EmscriptenBrowserBuild(Wasm32EmscriptenBuild): + buildersuffix = ".emscripten-browser" + extra_configure_flags = [ + "--with-emscripten-target=browser", + "--enable-wasm-dynamic-linking", + "--disable-wasm-pthreads", + ] + # browser builds do not accept argv from CLI + can_execute_python = False + + +class Wasm32WASIBuild(UnixCrossBuild): + """wasm32-wasi builder + + * WASI SDK >= 16 must be installed to default path /opt/wasi-sdk + * WASIX must be installed to /opt/wasix + * ccache must be installed + * wasmtime must be installed and on PATH + """ + buildersuffix = ".wasi" + factory_tags = ["wasm", "wasi"] + extra_configure_flags = [ + # debug builds exhaust the limited call stack on WASI + "--without-pydebug", + # ipv6 is not supported on WASI + "--disable-ipv6", + ] + wasi_sdk = "/opt/wasi-sdk" + wasi_sysroot = f"{wasi_sdk}/share/wasi-sysroot" + wasix = "/opt/wasix" + compile_environ = { + "CONFIG_SITE": "../../Tools/wasm/config.site-wasm32-wasi", + # use Clang from WASI-SDK + "CC": f"ccache {wasi_sdk}/bin/clang", + "LDSHARED": f"{wasi_sdk}/bin/wasm-ld", + "AR": f"{wasi_sdk}/bin/llvm-ar", + # use WASIX library with POSIX stubs + "CFLAGS": f"-isystem {wasix}/include", + "LDFLAGS": f"-L{wasix}/lib -lwasix", + # WASI-SDK does not have a 'wasm32-unknown-wasi-pkg-config' script + # force pkg-config into cross-compiling mode + "PKG_CONFIG_PATH": "", + "PKG_CONFIG_SYSROOT_DIR": wasi_sysroot, + "PKG_CONFIG_LIBDIR": f"{wasi_sysroot}/lib/pkgconfig:{wasi_sysroot}/share/pkgconfig", + } + host = "wasm32-unknown-wasi" + + def setup(self, parallel, branch, test_with_PTY=False, **kwargs): + self.addStep( + ShellCommand( + name="Touch srcdir Modules/Setup.local", + description="Hack to work around wasmtime mapdir issue", + command=["touch", "Modules/Setup.local"], + haltOnFailure=True, + ) + ) + super().setup( + parallel, branch, test_with_PTY=test_with_PTY, **kwargs + ) diff --git a/master/custom/workers.py b/master/custom/workers.py index aa3ff411f..77316cbb9 100644 --- a/master/custom/workers.py +++ b/master/custom/workers.py @@ -289,5 +289,10 @@ def get_workers(settings): tags=['windows', 'arm64'], parallel_tests=2, ), - + cpw( + name="bcannon-wasm", + tags=['wasm', 'emscripten', 'wasi'], + branches=['3.11', '3.x'], + parallel_tests=4, + ), ] diff --git a/master/master.cfg b/master/master.cfg index a64e72e63..b81c3e8bf 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -236,6 +236,14 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches): branch=branchname, **extra_factory_args.get(worker, {}), ) + tags = [branchname, stability,] + getattr(f, "tags", []) + if tier: + tags.append(tier) + + # Only 3.11+ for WebAssembly builds + if "wasm" in tags and branchname in {"3.8", "3.9", "3.10"}: + continue + if name in DAILYBUILDERS: dailybuildernames.append(buildername) else: