diff --git a/ChangeLog.md b/ChangeLog.md index 3f73144f72307..72bda3bec328c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -20,6 +20,9 @@ See docs/process.md for more on how version tagging works. Current Trunk ------------- +- Emscripten now builds a complete sysroot inside the EM_CACHE directory. + This includes the system headers which get copied into place there rather + than adding a sequence of extra include directories. 2.0.12: 01/09/2021 ------------------ diff --git a/embuilder.py b/embuilder.py index 75b62332ccac8..b9fa855cc763a 100755 --- a/embuilder.py +++ b/embuilder.py @@ -171,6 +171,10 @@ def main(): if force: library.erase() library.get_path() + elif what == 'sysroot': + if force: + shared.Cache.erase_file('sysroot_install.stamp') + system_libs.ensure_sysroot() elif what == 'struct_info': if force: shared.Cache.erase_file('generated_struct_info.json') diff --git a/emcc.py b/emcc.py index 251d61c618a2c..c5323a2c52b69 100755 --- a/emcc.py +++ b/emcc.py @@ -1331,12 +1331,6 @@ def filter_out_duplicate_dynamic_libs(inputs): '_emscripten_stack_get_end', '_emscripten_stack_set_limits'] - if not compile_only and not options.post_link: - ldflags = shared.emsdk_ldflags(newargs) - for f in ldflags: - newargs.append(f) - add_link_flag(len(newargs), f) - # SSEx is implemented on top of SIMD128 instruction set, but do not pass SSE flags to LLVM # so it won't think about generating native x86 SSE code. newargs = [x for x in newargs if x not in shared.SIMD_INTEL_FEATURE_TOWER and x not in shared.SIMD_NEON_FLAGS] @@ -1950,6 +1944,7 @@ def is_link_flag(flag): compile_args = [a for a in newargs if a and not is_link_flag(a)] cflags = calc_cflags(options) + system_libs.ensure_sysroot() system_libs.add_ports_cflags(cflags, shared.Settings) def use_cxx(src): @@ -1976,9 +1971,8 @@ def get_compiler(cxx): return CC def get_clang_command(src_file): - cxx = use_cxx(src_file) - per_file_cflags = shared.get_cflags(args, cxx) - return get_compiler(cxx) + cflags + per_file_cflags + compile_args + [src_file] + per_file_cflags = shared.get_cflags(args) + return get_compiler(use_cxx(src_file)) + cflags + per_file_cflags + compile_args + [src_file] def get_clang_command_asm(src_file): asflags = shared.get_clang_flags() @@ -2070,6 +2064,10 @@ def compile_source_file(i, input_file): if specified_target and specified_target.startswith('-'): exit_with_error('invalid output filename: `%s`' % specified_target) + ldflags = shared.emsdk_ldflags(newargs) + for f in ldflags: + add_link_flag(sys.maxsize, f) + using_lld = not (link_to_object and shared.Settings.LTO) link_flags = filter_link_flags(link_flags, using_lld) diff --git a/src/struct_info.json b/src/struct_info.json index d8736ebb25087..24e993c580ddd 100644 --- a/src/struct_info.json +++ b/src/struct_info.json @@ -1454,7 +1454,7 @@ ] }, { - "file": "../lib/libc/musl/src/internal/pthread_impl.h", + "file": "system/lib/libc/musl/src/internal/pthread_impl.h", "structs": { "pthread": [ "threadStatus", @@ -1477,7 +1477,7 @@ "defines": ["__ATTRP_C11_THREAD"] }, { - "file": "../lib/libc/musl/src/internal/libc.h", + "file": "system/lib/libc/musl/src/internal/libc.h", "structs": { "libc": [ "global_locale" diff --git a/system/lib/fetch/asmfs.cpp b/system/lib/fetch/asmfs.cpp index 05f8446ad6987..cde27108c1ea9 100644 --- a/system/lib/fetch/asmfs.cpp +++ b/system/lib/fetch/asmfs.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/tests/test_other.py b/tests/test_other.py index ab101357882c6..109bf9c47f4b3 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -480,10 +480,10 @@ def test_emcc_asm_v_wasm(self): def test_emcc_cflags(self): output = self.run_process([EMCC, '--cflags'], stdout=PIPE) flags = output.stdout.strip() - self.assertContained(shared.shlex_join(shared.emsdk_cflags([], False)), flags) + self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags) output = self.run_process([EMXX, '--cflags'], stdout=PIPE) flags = output.stdout.strip() - self.assertContained(shared.shlex_join(shared.emsdk_cflags([], True)), flags) + self.assertContained(shared.shlex_join(shared.emsdk_cflags([])), flags) # check they work cmd = [CLANG_CXX, path_from_root('tests', 'hello_world.cpp')] + shlex.split(flags.replace('\\', '\\\\')) + ['-c', '-o', 'out.o'] self.run_process(cmd) diff --git a/tests/test_sanity.py b/tests/test_sanity.py index 8c2179dd44d2e..8a8eef3c2e002 100644 --- a/tests/test_sanity.py +++ b/tests/test_sanity.py @@ -691,10 +691,10 @@ def test_embuilder_wasm_backend(self): # the --lto flag makes us build wasm-bc self.do([EMCC, '--clear-cache']) self.run_process([EMBUILDER, 'build', 'libemmalloc']) - self.assertExists(os.path.join(config.CACHE, 'wasm')) + self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten')) self.do([EMCC, '--clear-cache']) self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto']) - self.assertExists(os.path.join(config.CACHE, 'wasm-lto')) + self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lto')) def test_binaryen_version(self): restore_and_set_up() diff --git a/tools/cache.py b/tools/cache.py index 67cb8f60a03be..a43dec5bfb86e 100644 --- a/tools/cache.py +++ b/tools/cache.py @@ -84,21 +84,32 @@ def erase(self): def get_path(self, name): return os.path.join(self.dirname, name) + def get_sysroot_dir(self, absolute): + if absolute: + return os.path.join(self.dirname, 'sysroot') + return 'sysroot' + def get_include_dir(self): - return os.path.join(self.dirname, 'include') + return os.path.join(self.get_sysroot_dir(absolute=True), 'include') - def get_lib_dir(self): - subdir = 'wasm' + def get_lib_dir(self, absolute): + path = os.path.join(self.get_sysroot_dir(absolute=absolute), 'lib') + if shared.Settings.MEMORY64: + path = os.path.join(path, 'wasm64-emscripten') + else: + path = os.path.join(path, 'wasm32-emscripten') + # if relevant, use a subdir of the cache + subdir = [] if shared.Settings.LTO: - subdir += '-lto' + subdir.append('lto') if shared.Settings.RELOCATABLE: - subdir += '-pic' - if shared.Settings.MEMORY64: - subdir += '-memory64' - return subdir + subdir.append('pic') + if subdir: + path = os.path.join(path, '-'.join(subdir)) + return path def get_lib_name(self, name): - return os.path.join(self.get_lib_dir(), name) + return os.path.join(self.get_lib_dir(absolute=False), name) def erase_lib(self, name): self.erase_file(self.get_lib_name(name)) diff --git a/tools/gen_struct_info.py b/tools/gen_struct_info.py index 8bcba80c2fac2..d7e8987e3cfb3 100755 --- a/tools/gen_struct_info.py +++ b/tools/gen_struct_info.py @@ -246,6 +246,7 @@ def inspect_code(headers, cpp_opts, structs, defines): '-O0', '-Werror', '-Wno-format', + '-I', shared.path_from_root(), '-s', 'BOOTSTRAPPING_STRUCT_INFO=1', '-s', 'WARN_ON_UNDEFINED_SYMBOLS=0', '-s', 'STRICT=1', diff --git a/tools/ports/regal.py b/tools/ports/regal.py index 81e398ca3408d..2c7eee743de93 100644 --- a/tools/ports/regal.py +++ b/tools/ports/regal.py @@ -46,7 +46,7 @@ def create(): # includes source_path_include = os.path.join(ports.get_dir(), 'regal', 'regal-' + TAG, 'include', 'GL') - ports.install_header_dir(source_path_include) + ports.install_headers(source_path_include, target='GL') # build srcs_regal = ['regal/RegalShaderInstance.cpp', diff --git a/tools/shared.py b/tools/shared.py index 74666ee66fb47..6982adcd0d9f6 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -420,9 +420,7 @@ def emsdk_ldflags(user_args): return [] library_paths = [ - path_from_root('system', 'local', 'lib'), - path_from_root('system', 'lib'), - Cache.get_path(Cache.get_lib_dir()) + Cache.get_lib_dir(absolute=True) ] ldflags = ['-L' + l for l in library_paths] @@ -437,12 +435,12 @@ def emsdk_ldflags(user_args): return ldflags -def emsdk_cflags(user_args, cxx): +def emsdk_cflags(user_args): # Disable system C and C++ include directories, and add our own (using # -isystem so they are last, like system dirs, which allows projects to # override them) - c_opts = ['--sysroot=' + path_from_root('system')] + c_opts = ['--sysroot=' + Cache.get_sysroot_dir(absolute=True)] def array_contains_any_of(hay, needles): for n in needles: @@ -477,24 +475,13 @@ def array_contains_any_of(hay, needles): sysroot_include_paths = [] - if cxx: - sysroot_include_paths += [ - os.path.join('/include', 'libcxx'), - os.path.join('/lib', 'libcxxabi', 'include'), - ] - - # TODO: Merge the cache into the sysroot. - c_opts += ['-Xclang', '-isystem' + Cache.get_path('include')] - sysroot_include_paths += [ os.path.join('/include', 'compat'), - os.path.join('/include', 'libc'), - os.path.join('/lib', 'libc', 'musl', 'arch', 'emscripten'), - os.path.join('/local', 'include'), + # TODO(sbc): Ideally we wouldn't need these, we could just copy them into the sysroot/include + # However, clang puts its internal header directory first so it finds its internal versions + # first. os.path.join('/include', 'SSE'), os.path.join('/include', 'neon'), - os.path.join('/lib', 'compiler-rt', 'include'), - os.path.join('/lib', 'libunwind', 'include'), ] def include_directive(paths): @@ -511,7 +498,7 @@ def get_clang_flags(): return ['-target', get_llvm_target()] -def get_cflags(user_args, cxx): +def get_cflags(user_args): c_opts = get_clang_flags() # Set the LIBCPP ABI version to at least 2 so that we get nicely aligned string @@ -539,7 +526,7 @@ def get_cflags(user_args, cxx): if os.environ.get('EMMAKEN_NO_SDK') or '-nostdinc' in user_args: return c_opts - return c_opts + emsdk_cflags(user_args, cxx) + return c_opts + emsdk_cflags(user_args) # Settings. A global singleton. Not pretty, but nicer than passing |, settings| everywhere diff --git a/tools/system_libs.py b/tools/system_libs.py index 54d16c3a60943..2d5a82de8eb6b 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -83,6 +83,10 @@ def run_one_command(cmd): def run_build_commands(commands): + # Before running a set of build commands make sure the common sysroot + # headers are installed. This prevents each sub-process from attempting + # to setup the sysroot itself. + ensure_sysroot() cores = min(len(commands), building.get_num_cores()) if cores <= 1: for command in commands: @@ -952,6 +956,9 @@ class libunwind(NoExceptLibrary, MTLibrary): name = 'libunwind' cflags = ['-Oz', '-D_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS'] src_dir = ['system', 'lib', 'libunwind', 'src'] + # Without this we can't build libunwind since it will pickup the unwind.h + # that is part of llvm (which is not compatible for some reason). + includes = [['system', 'lib', 'libunwind', 'include']] src_files = ['Unwind-wasm.cpp'] def __init__(self, **kwargs): @@ -1883,7 +1890,7 @@ def build_port(port_name, settings): def add_ports_cflags(args, settings): # Legacy SDL1 port is not actually a port at all but builtin if settings.USE_SDL == 1: - args += ['-Xclang', '-isystem' + shared.path_from_root('system', 'include', 'SDL')] + args += ['-Xclang', '-iwithsysroot/include/SDL'] needed = get_needed_ports(settings) @@ -1900,3 +1907,51 @@ def show_ports(): print('Available ports:') for port in ports.ports: print(' ', port.show()) + + +# Once we require python 3.8 we can use shutil.copytree with +# dirs_exist_ok=True and remove this function. +def copytree_exist_ok(src, dest): + with utils.chdir(src): + for dirname, dirs, files in os.walk('.'): + destdir = os.path.join(dest, dirname) + utils.safe_ensure_dirs(destdir) + for f in files: + shared.safe_copy(os.path.join(src, dirname, f), os.path.join(destdir, f)) + + +def install_system_headers(): + install_dirs = { + ('include',): '', + ('lib', 'compiler-rt', 'include'): '', + ('include', 'compat'): 'compat', + ('lib', 'libunwind', 'include'): '', + ('lib', 'libc', 'musl', 'arch', 'emscripten'): '', + ('lib', 'libcxx'): '', + ('include', 'libc'): '', + ('include', 'libcxx'): os.path.join('c++', 'v1'), + ('lib', 'libcxxabi', 'include'): os.path.join('c++', 'v1'), + } + + target_include_dir = shared.Cache.get_include_dir() + for src, dest in install_dirs.items(): + src = shared.path_from_root('system', *src) + dest = os.path.join(target_include_dir, dest) + copytree_exist_ok(src, dest) + + # TODO(sbc): Move these headers back into thier respecive source trees + for dirname in ['libc', 'libcxx']: + shutil.rmtree(os.path.join(target_include_dir, dirname)) + + # Create a stamp file that signal the the header have been installed + # Removing this file, or running `emcc --clear-cache` or running + # `./embuilder build sysroot --force` will cause the re-installation of + # the system headers. + stamp = shared.Cache.get_path('sysroot_install.stamp') + with open(stamp, 'w') as f: + f.write('x') + return stamp + + +def ensure_sysroot(): + shared.Cache.get('sysroot_install.stamp', install_system_headers, what='system headers')