From a74f3c66d42001914d9f63ea2b6313c7cf57972c Mon Sep 17 00:00:00 2001 From: Qijia Liu Date: Wed, 15 Jan 2025 18:12:17 -0500 Subject: [PATCH 1/3] Always treat host FS as case-insensitive to produce same libc cross OS --- tools/system_libs.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index ea06d88515a74..aa37c5609d578 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -219,7 +219,6 @@ def create_ninja_file(input_files, filename, libname, cflags, asflags=None, cust suffix = shared.suffix(libname) build_dir = os.path.dirname(filename) - case_insensitive = is_case_insensitive(os.path.dirname(filename)) if suffix == '.o': assert len(input_files) == 1 input_file = escape_ninja_path(input_files[0]) @@ -232,9 +231,8 @@ def create_ninja_file(input_files, filename, libname, cflags, asflags=None, cust # Resolve duplicates by appending unique. # This is needed on case insensitive filesystem to handle, # for example, _exit.o and _Exit.o. - object_basename = shared.unsuffixed_basename(src) - if case_insensitive: - object_basename = object_basename.lower() + # Always apply it to produce same libraries on different filesystems. + object_basename = shared.unsuffixed_basename(src).lower() o = os.path.join(build_dir, object_basename + '.o') object_uuid = 0 # Find a unique basename @@ -270,14 +268,6 @@ def create_ninja_file(input_files, filename, libname, cflags, asflags=None, cust ensure_target_in_ninja_file(get_top_level_ninja_file(), f'subninja {escape_ninja_path(filename)}') -def is_case_insensitive(path): - """Returns True if the filesystem at `path` is case insensitive.""" - utils.write_file(os.path.join(path, 'test_file'), '') - case_insensitive = os.path.exists(os.path.join(path, 'TEST_FILE')) - os.remove(os.path.join(path, 'test_file')) - return case_insensitive - - class Library: """ `Library` is the base class of all system libraries. @@ -490,7 +480,6 @@ def build_objects(self, build_dir): commands = [] objects = set() cflags = self.get_cflags() - case_insensitive = is_case_insensitive(build_dir) for src in self.get_files(): ext = shared.suffix(src) if ext in {'.s', '.S', '.c'}: @@ -507,15 +496,12 @@ def build_objects(self, build_dir): cmd += cflags cmd = self.customize_build_cmd(cmd, src) - object_basename = shared.unsuffixed_basename(src) - if case_insensitive: - object_basename = object_basename.lower() + object_basename = shared.unsuffixed_basename(src).lower() o = os.path.join(build_dir, object_basename + '.o') if o in objects: - # If we have seen a file with the same name before, we are on a case-insensitive - # filesystem and need a separate command to compile this file with a - # custom unique output object filename, as batch compile doesn't allow - # such customization. + # If we have seen a file with the same name before, we need a separate + # command to compile this file with a custom unique output object + # filename, as batch compile doesn't allow such customization. # # This is needed to handle, for example, _exit.o and _Exit.o. object_uuid = 0 From b1acfc69477d1210275796844268824b8607e0b5 Mon Sep 17 00:00:00 2001 From: Qijia Liu Date: Wed, 15 Jan 2025 23:35:53 -0500 Subject: [PATCH 2/3] fix embuilder build libunwind --- tools/system_libs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index aa37c5609d578..f955bc6667f30 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -510,15 +510,18 @@ def build_objects(self, build_dir): object_uuid += 1 o = os.path.join(build_dir, f'{object_basename}__{object_uuid}.o') commands.append(cmd + [src, '-o', o]) + objects.add(o) elif batch_inputs: # Use relative paths to reduce the length of the command line. # This allows to avoid switching to a response file as often. src = os.path.relpath(src, build_dir) src = utils.normalize_path(src) batches.setdefault(tuple(cmd), []).append(src) + # No -o in command, use original file name. + objects.add(os.path.join(build_dir, shared.unsuffixed_basename(src) + '.o')) else: commands.append(cmd + [src, '-o', o]) - objects.add(o) + objects.add(o) if batch_inputs: # Choose a chunk size that is large enough to avoid too many subprocesses From b8021471dc3bfb8ae3049a288e67f67b6cda8005 Mon Sep 17 00:00:00 2001 From: Qijia Liu Date: Thu, 16 Jan 2025 17:56:59 -0500 Subject: [PATCH 3/3] simplify; comment --- tools/system_libs.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index f955bc6667f30..4adeeac5c9b5b 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -228,10 +228,10 @@ def create_ninja_file(input_files, filename, libname, cflags, asflags=None, cust else: objects = [] for src in input_files: - # Resolve duplicates by appending unique. - # This is needed on case insensitive filesystem to handle, - # for example, _exit.o and _Exit.o. - # Always apply it to produce same libraries on different filesystems. + # Resolve duplicates by appending unique. This is needed on case + # insensitive filesystem to handle, for example, _exit.o and _Exit.o. + # This is done even on case sensitive filesystem so that builds are + # reproducible across platforms. object_basename = shared.unsuffixed_basename(src).lower() o = os.path.join(build_dir, object_basename + '.o') object_uuid = 0 @@ -510,7 +510,6 @@ def build_objects(self, build_dir): object_uuid += 1 o = os.path.join(build_dir, f'{object_basename}__{object_uuid}.o') commands.append(cmd + [src, '-o', o]) - objects.add(o) elif batch_inputs: # Use relative paths to reduce the length of the command line. # This allows to avoid switching to a response file as often. @@ -518,10 +517,10 @@ def build_objects(self, build_dir): src = utils.normalize_path(src) batches.setdefault(tuple(cmd), []).append(src) # No -o in command, use original file name. - objects.add(os.path.join(build_dir, shared.unsuffixed_basename(src) + '.o')) + o = os.path.join(build_dir, shared.unsuffixed_basename(src) + '.o') else: commands.append(cmd + [src, '-o', o]) - objects.add(o) + objects.add(o) if batch_inputs: # Choose a chunk size that is large enough to avoid too many subprocesses