change tar command used in get_source_tarball_from_git to get reproducible tarballs

easybuild/tools/filetools.py

@@ -2690,11 +2690,15 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
             for cmd in cmds:
                 run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name, trace=False)
 
-    # create an archive and delete the git repo directory
-    if keep_git_dir:
-        tar_cmd = ['tar', 'cfvz', targetpath, repo_name]
-    else:
-        tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
+    # When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag
+    # '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and
+    # --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/
+    tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|',
+               'LC_ALL=C', 'sort', '--zero-terminated', '|',
+               'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion',
+               '--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0',
+               '--numeric-owner', '--format=gnu', '--null',
+               '--no-recursion', '--files-from -']
     run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False, trace=False)
 
     # cleanup (repo_name dir does not exist in dry run mode)

test/framework/filetools.py

@@ -2787,65 +2787,87 @@ def run_check():
             'url': '[email protected]:easybuilders',
             'tag': 'tag_for_tests',
         }
-        git_repo = {'git_repo': '[email protected]:easybuilders/testrepository.git'}  # Just to make the below shorter
+        string_args = {
+            'git_repo': '[email protected]:easybuilders/testrepository.git',
+            'test_prefix': self.test_prefix,
+        }
+
         expected = '\n'.join([
             r'  running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"',
-            r"  \(in /.*\)",
-            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            r"  \(in /.*\)",
-        ]) % git_repo
+            r"  \(in .*/tmp.*\)",
+            r'  running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
+            r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz  --no-recursion'
+            r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
+            r' --null --no-recursion --files-from -"',
+            r"  \(in .*/tmp.*\)",
+        ]) % string_args
         run_check()
 
         git_config['clone_into'] = 'test123'
         expected = '\n'.join([
             r'  running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"',
-            r"  \(in /.*\)",
-            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"',
-            r"  \(in /.*\)",
-        ]) % git_repo
+            r"  \(in .*/tmp.*\)",
+            r'  running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
+            r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
+            r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
+            r' --null --no-recursion --files-from -"',
+            r"  \(in .*/tmp.*\)",
+        ]) % string_args
         run_check()
         del git_config['clone_into']
 
         git_config['recursive'] = True
         expected = '\n'.join([
             r'  running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"',
-            r"  \(in /.*\)",
-            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            r"  \(in /.*\)",
-        ]) % git_repo
+            r"  \(in .*/tmp.*\)",
+            r'  running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
+            r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
+            r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
+            r' --null --no-recursion --files-from -"',
+            r"  \(in .*/tmp.*\)",
+        ]) % string_args
         run_check()
 
         git_config['keep_git_dir'] = True
         expected = '\n'.join([
             r'  running command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
-            r"  \(in /.*\)",
-            r'  running command "tar cfvz .*/target/test.tar.gz testrepository"',
-            r"  \(in /.*\)",
-        ]) % git_repo
+            r"  \(in .*/tmp.*\)",
+            r'  running command "find testrepository -print0  | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar'
+            r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip'
+            r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion'
+            r' --files-from -"',
+            r"  \(in .*/tmp.*\)",
+        ]) % string_args
         run_check()
         del git_config['keep_git_dir']
 
         del git_config['tag']
         git_config['commit'] = '8456f86'
         expected = '\n'.join([
             r'  running command "git clone --no-checkout %(git_repo)s"',
-            r"  \(in /.*\)",
+            r"  \(in .*/tmp.*\)",
             r'  running command "git checkout 8456f86 && git submodule update --init --recursive"',
             r"  \(in testrepository\)",
-            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            r"  \(in /.*\)",
-        ]) % git_repo
+            r'  running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
+            r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
+            r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
+            r' --null --no-recursion --files-from -"',
+            r"  \(in .*/tmp.*\)",
+        ]) % string_args
         run_check()
 
         del git_config['recursive']
         expected = '\n'.join([
             r'  running command "git clone --no-checkout %(git_repo)s"',
-            r"  \(in /.*\)",
+            r"  \(in .*/tmp.*\)",
             r'  running command "git checkout 8456f86"',
             r"  \(in testrepository\)",
-            r'  running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
-            r"  \(in /.*\)",
-        ]) % git_repo
+            r'  running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
+            r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
+            r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
+            r' --null --no-recursion --files-from -"',
+            r"  \(in .*/tmp.*\)",
+        ]) % string_args
         run_check()
 
         # Test with real data.