Skip to content

determine file_info for all easyconfigs before any actual copying #2604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
boegel merged 2 commits into from
Oct 2, 2018
Merged

determine file_info for all easyconfigs before any actual copying #2604

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
91bd9da
flesh out code to determine file_info in copy_easyconfigs
migueldiascosta Oct 2, 2018
aa2a4b9
fix lines too long
migueldiascosta Oct 2, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions easybuild/framework/easyconfig/easyconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1613,16 +1613,17 @@ def clean_up_easyconfigs(paths):
write_file(path, ectxt, forced=True)


def copy_easyconfigs(paths, target_dir):
def det_file_info(paths, target_dir):
"""
Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot.
Determine useful information on easyconfig files relative to a target directory, before any actual operation (e.g. copying) is performed

:param paths: list of paths to copy to git working dir
:param paths: list of paths to easyconfig files
:param target_dir: target directory
:return: dict with useful information on copied easyconfig files (corresponding EasyConfig instances, paths, status)
:return: dict with useful information on easyconfig files (corresponding EasyConfig instances, paths, status) relative to a target directory
"""
file_info = {
'ecs': [],
'paths': [],
'paths_in_repo': [],
'new': [],
'new_folder': [],
Expand All @@ -1632,6 +1633,7 @@ def copy_easyconfigs(paths, target_dir):
for path in paths:
ecs = process_easyconfig(path, validate=False)
if len(ecs) == 1:
file_info['paths'].append(path)
file_info['ecs'].append(ecs[0]['ec'])

soft_name = file_info['ecs'][-1].name
Expand All @@ -1644,14 +1646,27 @@ def copy_easyconfigs(paths, target_dir):
file_info['new'].append(new_file)
file_info['new_folder'].append(new_folder)
file_info['new_file_in_existing_folder'].append(new_file and not new_folder)

copy_file(path, target_path, force_in_dry_run=True)

file_info['paths_in_repo'].append(target_path)

else:
raise EasyBuildError("Multiple EasyConfig instances obtained from easyconfig file %s", path)

return file_info


def copy_easyconfigs(paths, target_dir):
"""
Copy easyconfig files to specified directory, in the 'right' location and using the filename expected by robot.

:param paths: list of paths to copy to git working dir
:param target_dir: target directory
:return: dict with useful information on copied easyconfig files (corresponding EasyConfig instances, paths, status)
"""
file_info = det_file_info(paths, target_dir)

for path, target_path in zip(file_info['paths'], file_info['paths_in_repo']):
copy_file(path, target_path, force_in_dry_run=True)

if build_option('cleanup_easyconfigs'):
clean_up_easyconfigs(file_info['paths_in_repo'])

Expand Down
6 changes: 3 additions & 3 deletions test/framework/easyconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1851,7 +1851,7 @@ def test_copy_easyconfigs(self):
# passing an empty list of paths is fine
res = copy_easyconfigs([], target_dir)
self.assertEqual(res, {'ecs': [], 'new': [], 'new_file_in_existing_folder': [],
'new_folder': [], 'paths_in_repo': []})
'new_folder': [], 'paths': [], 'paths_in_repo': []})
self.assertEqual(os.listdir(ecs_target_dir), [])

# copy test easyconfigs, purposely under a different name
Expand All @@ -1868,7 +1868,7 @@ def test_copy_easyconfigs(self):

res = copy_easyconfigs(ecs_to_copy, target_dir)
self.assertEqual(sorted(res.keys()), ['ecs', 'new', 'new_file_in_existing_folder',
'new_folder', 'paths_in_repo'])
'new_folder', 'paths', 'paths_in_repo'])
self.assertEqual(len(res['ecs']), len(test_ecs))
self.assertTrue(all(isinstance(ec, EasyConfig) for ec in res['ecs']))
self.assertTrue(all(res['new']))
Expand Down Expand Up @@ -1900,7 +1900,7 @@ def test_copy_easyconfigs(self):
# copy single easyconfig with buildstats included for running further tests
res = copy_easyconfigs([toy_ec], target_dir)

self.assertEqual([len(x) for x in res.values()], [1, 1, 1, 1, 1])
self.assertEqual([len(x) for x in res.values()], [1, 1, 1, 1, 1, 1])
self.assertEqual(res['ecs'][0].full_mod_name, 'toy/0.0')

# toy-0.0.eb was already copied into target_dir, so should not be marked as new anymore
Expand Down