Skip to content

Commit bf4f3f6

Browse files
RovanionRovanion
committed
Build reproducible tarballs from git
1 parent f725e61 commit bf4f3f6

File tree

2 files changed

+44
-18
lines changed

2 files changed

+44
-18
lines changed

‎easybuild/tools/filetools.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2707,11 +2707,15 @@ def get_source_tarball_from_git(filename, targetdir, git_config):
27072707
for cmd in cmds:
27082708
run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name)
27092709

2710-
# create an archive and delete the git repo directory
2711-
if keep_git_dir:
2712-
tar_cmd = ['tar', 'cfvz', targetpath, repo_name]
2713-
else:
2714-
tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name]
2710+
# When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag
2711+
# '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and
2712+
# --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/
2713+
tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|',
2714+
'LC_ALL=C', 'sort', '--zero-terminated', '|',
2715+
'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion',
2716+
'--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0',
2717+
'--numeric-owner', '--format=gnu', '--null',
2718+
'--no-recursion', '--files-from -']
27152719
run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False)
27162720

27172721
# cleanup (repo_name dir does not exist in dry run mode)

‎test/framework/filetools.py‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2780,41 +2780,57 @@ def run_check():
27802780
'url': '[email protected]:easybuilders',
27812781
'tag': 'tag_for_tests',
27822782
}
2783-
git_repo = {'git_repo': '[email protected]:easybuilders/testrepository.git'} # Just to make the below shorter
2783+
string_args = {
2784+
'git_repo': '[email protected]:easybuilders/testrepository.git',
2785+
'test_prefix': self.test_prefix,
2786+
}
2787+
27842788
expected = '\n'.join([
27852789
r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"',
27862790
r" \(in .*/tmp.*\)",
2787-
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
2791+
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
2792+
r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion'
2793+
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
2794+
r' --null --no-recursion --files-from -"',
27882795
r" \(in .*/tmp.*\)",
2789-
]) % git_repo
2796+
]) % string_args
27902797
run_check()
27912798

27922799
git_config['clone_into'] = 'test123'
27932800
expected = '\n'.join([
27942801
r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"',
27952802
r" \(in .*/tmp.*\)",
2796-
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git test123"',
2803+
r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
2804+
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
2805+
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
2806+
r' --null --no-recursion --files-from -"',
27972807
r" \(in .*/tmp.*\)",
2798-
]) % git_repo
2808+
]) % string_args
27992809
run_check()
28002810
del git_config['clone_into']
28012811

28022812
git_config['recursive'] = True
28032813
expected = '\n'.join([
28042814
r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"',
28052815
r" \(in .*/tmp.*\)",
2806-
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
2816+
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
2817+
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
2818+
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
2819+
r' --null --no-recursion --files-from -"',
28072820
r" \(in .*/tmp.*\)",
2808-
]) % git_repo
2821+
]) % string_args
28092822
run_check()
28102823

28112824
git_config['keep_git_dir'] = True
28122825
expected = '\n'.join([
28132826
r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"',
28142827
r" \(in .*/tmp.*\)",
2815-
r' running command "tar cfvz .*/target/test.tar.gz testrepository"',
2828+
r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar'
2829+
r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip'
2830+
r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion'
2831+
r' --files-from -"',
28162832
r" \(in .*/tmp.*\)",
2817-
]) % git_repo
2833+
]) % string_args
28182834
run_check()
28192835
del git_config['keep_git_dir']
28202836

@@ -2825,9 +2841,12 @@ def run_check():
28252841
r" \(in .*/tmp.*\)",
28262842
r' running command "git checkout 8456f86 && git submodule update --init --recursive"',
28272843
r" \(in testrepository\)",
2828-
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
2844+
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
2845+
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
2846+
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
2847+
r' --null --no-recursion --files-from -"',
28292848
r" \(in .*/tmp.*\)",
2830-
]) % git_repo
2849+
]) % string_args
28312850
run_check()
28322851

28332852
del git_config['recursive']
@@ -2836,9 +2855,12 @@ def run_check():
28362855
r" \(in .*/tmp.*\)",
28372856
r' running command "git checkout 8456f86"',
28382857
r" \(in testrepository\)",
2839-
r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"',
2858+
r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated'
2859+
r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion'
2860+
r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu'
2861+
r' --null --no-recursion --files-from -"',
28402862
r" \(in .*/tmp.*\)",
2841-
]) % git_repo
2863+
]) % string_args
28422864
run_check()
28432865

28442866
# Test with real data.

0 commit comments

Comments
 (0)